{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "86a4bd36",
   "metadata": {},
   "source": [
    "\n",
    "\n",
    "下面展示基于隐马尔科夫模型的序列标注监督学习的代码。这里以命名实体识别任务为例，所使用的数据是Books数据集。为简单起见，标签序列采用BIO格式。首先构建数据集和标签集合："
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "41673bbb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'B-NAME': 1, 'I-ORG': 2, 'B-PRO': 3, 'B-EDU': 4, 'I-NAME': 5, 'B-LOC': 6, 'B-TITLE': 7, 'B-RACE': 8, 'I-TITLE': 9, 'I-RACE': 10, 'I-PRO': 11, 'B-CONT': 12, 'I-EDU': 13, 'I-LOC': 14, 'I-CONT': 15, 'B-ORG': 16, 'O': 0}\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import json\n",
    "from collections import defaultdict\n",
    "\n",
    "\n",
    "class NERDataset:\n",
    "    def __init__(self):\n",
    "        train_file, test_file, label_file = 'ner_train.jsonl',\\\n",
    "            'ner_test.jsonl', 'ner_labels.json'\n",
    "        \n",
    "        def read_file(file_name):\n",
    "            with open(file_name, 'r', encoding='utf-8') as fin:\n",
    "                json_list = list(fin)\n",
    "            data_split = []\n",
    "            for json_str in json_list:\n",
    "                raw = json.loads(json_str)\n",
    "                d = {'tokens': raw['tokens'], 'tags': raw['tags']}\n",
    "                data_split.append(d)\n",
    "            return data_split\n",
    "        \n",
    "        # 读取JSON文件，转化为Python对象\n",
    "        self.train_data, self.test_data = read_file(train_file),\\\n",
    "            read_file(test_file)\n",
    "        self.label2id = json.loads(open(label_file, 'r').read())\n",
    "        self.id2label = {}\n",
    "        for k, v in self.label2id.items():\n",
    "            self.id2label[v] = k\n",
    "        print(self.label2id)\n",
    "\n",
    "    # 建立词表，过滤低频词\n",
    "    def build_vocab(self, min_freq=3):\n",
    "        # 统计词频\n",
    "        frequency = defaultdict(int)\n",
    "        for data in self.train_data:\n",
    "            tokens = data['tokens']\n",
    "            for token in tokens:\n",
    "                frequency[token] += 1\n",
    "                \n",
    "        print(f'unique tokens = {len(frequency)}, '+\\\n",
    "              f'total counts = {sum(frequency.values())}, '+\\\n",
    "              f'max freq = {max(frequency.values())}, '+\\\n",
    "              f'min freq = {min(frequency.values())}')\n",
    "        \n",
    "        # 由于词与标签一一对应，不能随便删除字符，\n",
    "        # 因此加入<unk>用于替代未登录词，加入<pad>用于批量训练\n",
    "        self.token2id = {'<pad>': 0, '<unk>': 1}\n",
    "        self.id2token = {0: '<pad>', 1: '<unk>'}\n",
    "        total_count = 0\n",
    "        # 将高频词加入词表\n",
    "        for token, freq in sorted(frequency.items(),\\\n",
    "                key=lambda x: -x[1]):\n",
    "            if freq > min_freq:\n",
    "                self.token2id[token] = len(self.token2id)\n",
    "                self.id2token[len(self.id2token)] = token\n",
    "                total_count += freq\n",
    "            else:\n",
    "                break\n",
    "        print(f'min_freq = {min_freq}, '\n",
    "              f'remaining tokens = {len(self.token2id)}, '\n",
    "              f'in-vocab rate = {total_count / sum(frequency.values())}')\n",
    "\n",
    "    # 将文本输入转化为词表中对应的索引\n",
    "    def convert_tokens_to_ids(self):\n",
    "        for data_split in [self.train_data, self.test_data]:\n",
    "            for data in data_split:\n",
    "                data['token_ids'] = []\n",
    "                for token in data['tokens']:\n",
    "                    data['token_ids'].append(self.token2id.get(token, 1)) \n",
    "        \n",
    "dataset = NERDataset()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "25650a7c",
   "metadata": {},
   "source": [
    "接下来建立词表，将词元转换为索引，并将数据转化为适合训练的格式。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a3eb49e5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "unique tokens = 2454, total counts = 182068, max freq = 5630, min freq = 1\n",
      "min_freq = 0, remaining tokens = 2456, in-vocab rate = 1.0\n",
      "[595, 510, 353, 263, 424, 225, 764, 637, 353, 65, 80, 47, 41, 74, 53, 41, 141, 23, 74, 7, 259, 27, 135, 47, 31, 74, 67, 25, 1605, 815, 1157, 225, 595, 1158, 738, 688, 353, 1025, 65, 234, 27, 135, 316, 41, 31, 7, 80, 41, 42, 31, 47, 27, 528, 41, 141, 27, 135, 67, 1005, 1606, 363, 19, 677, 294, 37, 678, 1756, 604, 873, 1006, 1005, 353, 33, 30, 9, 113, 5, 45, 11, 82, 60, 21, 26, 5, 186, 153, 100, 565, 873, 353, 424, 1993, 387, 346, 250, 13, 5, 102, 214, 25, 11, 82, 60, 21, 26, 5, 186, 2, 7, 459, 97, 89, 147, 140, 139, 69, 46, 21, 12, 2, 7, 964, 98, 240, 677, 59, 9, 103, 11, 82, 60, 335, 200, 76, 26, 6, 110, 20, 9, 15, 4, 122, 539, 241, 99, 621, 40, 200, 57, 82, 156, 25, 11, 82, 381, 371, 91, 202, 6, 52, 2, 7, 670, 100, 84, 204, 53, 120, 27, 42, 3, 56, 1159, 3, 329, 31, 265, 31, 3, 56, 394, 394, 3, 84, 357, 84, 7, 25, 397, 7, 41, 7, 74, 7, 135, 7, 31, 7, 87, 7, 259, 7, 31, 7, 74, 7, 41, 7, 131, 7, 28, 289, 385, 4]\n",
      "['阿', '里', '斯', '提', '德', '·', '波', '拉', '斯', '（', 'A', 'r', 'i', 's', 't', 'i', 'd', 'e', 's', ' ', 'B', 'o', 'u', 'r', 'a', 's', '）', '和', '卢', '卡', '雅', '·', '阿', '伊', '纳', '罗', '斯', '托', '（', 'L', 'o', 'u', 'k', 'i', 'a', ' ', 'A', 'i', 'n', 'a', 'r', 'o', 'z', 'i', 'd', 'o', 'u', '）', '夫', '妇', '二', '人', '均', '拥', '有', '希', '腊', '比', '雷', '埃', '夫', '斯', '技', '术', '教', '育', '学', '院', '计', '算', '机', '工', '程', '学', '位', '以', '及', '色', '雷', '斯', '德', '谟', '克', '利', '特', '大', '学', '电', '子', '和', '计', '算', '机', '工', '程', '学', '位', '，', ' ', '都', '从', '事', '过', '软', '件', '开', '发', '工', '作', '，', ' ', '且', '目', '前', '均', '为', '教', '授', '计', '算', '机', '相', '关', '课', '程', '的', '高', '中', '教', '师', '。', '他', '们', '写', '了', '很', '多', '关', '于', '算', '法', '和', '计', '算', '思', '维', '方', '面', '的', '书', '，', ' ', '涉', '及', 'P', 'y', 't', 'h', 'o', 'n', '、', 'C', '#', '、', 'J', 'a', 'v', 'a', '、', 'C', '+', '+', '、', 'P', 'H', 'P', ' ', '和', 'V', ' ', 'i', ' ', 's', ' ', 'u', ' ', 'a', ' ', 'l', ' ', 'B', ' ', 'a', ' ', 's', ' ', 'i', ' ', 'c', ' ', '等', '语', '言', '。']\n"
     ]
    }
   ],
   "source": [
    "# 截取一部分数据便于更快训练\n",
    "dataset.train_data = dataset.train_data[:1000]\n",
    "dataset.test_data = dataset.test_data[:200]\n",
    "\n",
    "dataset.build_vocab(min_freq=0)\n",
    "dataset.convert_tokens_to_ids()\n",
    "print(dataset.train_data[0]['token_ids'])\n",
    "print([dataset.id2token[token_id] for token_id in \\\n",
    "       dataset.train_data[0]['token_ids']])\n",
    "\n",
    "def collect_data(data_split):\n",
    "    X, Y = [], []\n",
    "    for data in data_split:\n",
    "        assert len(data['token_ids']) == len(data['tags'])\n",
    "        X.append(data['token_ids'])\n",
    "        Y.append(data['tags'])\n",
    "    return X, Y\n",
    "\n",
    "train_X, train_Y = collect_data(dataset.train_data)\n",
    "test_X, test_Y = collect_data(dataset.test_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7de17192",
   "metadata": {},
   "source": [
    "随后实现隐马尔科夫模型，使用最大似然估计得到模型参数，使用维特比算法进行解码。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "3229fd6a",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "class HMM:\n",
    "    def __init__(self, n_tags, n_tokens):\n",
    "        self.n_tags = n_tags\n",
    "        self.n_tokens = n_tokens\n",
    "    \n",
    "    # 使用最大似然估计计算模型参数\n",
    "    def fit(self, X, Y):\n",
    "        Y0_cnt = np.zeros(self.n_tags)\n",
    "        YY_cnt = np.zeros((self.n_tags, self.n_tags))\n",
    "        YX_cnt = np.zeros((self.n_tags, self.n_tokens))\n",
    "        for x, y in zip(X, Y):\n",
    "            Y0_cnt[y[0]] += 1\n",
    "            last_y = y[0]\n",
    "            for i in range(1, len(y)):\n",
    "                YY_cnt[last_y, y[i]] += 1\n",
    "                last_y = y[i]\n",
    "            for xi, yi in zip(x, y):\n",
    "                YX_cnt[yi, xi] += 1\n",
    "        self.init_prob = Y0_cnt / Y0_cnt.sum()\n",
    "        self.transition_prob = YY_cnt\n",
    "        self.emission_prob = YX_cnt\n",
    "        for i in range(self.n_tags):\n",
    "            # 为了避免训练集过小时除0\n",
    "            yy_sum = YY_cnt[i].sum()\n",
    "            if yy_sum > 0:\n",
    "                self.transition_prob[i] = YY_cnt[i] / yy_sum\n",
    "            yx_sum = YX_cnt[i].sum()\n",
    "            if yx_sum > 0:\n",
    "                self.emission_prob[i] = YX_cnt[i] / yx_sum\n",
    "    \n",
    "    # 已知模型参数的条件下，使用维特比算法解码得到最优标签序列\n",
    "    def viterbi(self, x):\n",
    "        assert hasattr(self, 'init_prob') and hasattr(self,\\\n",
    "            'transition_prob') and hasattr(self, 'emission_prob')\n",
    "        Pi = np.zeros((len(x), self.n_tags))\n",
    "        Y = np.zeros((len(x), self.n_tags), dtype=np.int32)\n",
    "        # 初始化\n",
    "        for i in range(self.n_tags):\n",
    "            Pi[0, i] = self.init_prob[i] * self.emission_prob[i, x[0]]\n",
    "            Y[0, i] = -1\n",
    "        for t in range(1, len(x)):\n",
    "            for i in range(self.n_tags):\n",
    "                tmp = []\n",
    "                for j in range(self.n_tags):\n",
    "                    tmp.append(self.transition_prob[j, i] * Pi[t-1, j])\n",
    "                best_j = np.argmax(tmp)\n",
    "                # 维特比算法递推公式\n",
    "                Pi[t, i] = self.emission_prob[i, x[t]] * tmp[best_j]\n",
    "                Y[t, i] = best_j\n",
    "        y = [np.argmax(Pi[-1])]\n",
    "        for t in range(len(x)-1, 0, -1):\n",
    "            y.append(Y[t, y[-1]])\n",
    "        return np.max(Pi[len(x)-1]), y[::-1]\n",
    "    \n",
    "    def decode(self, X):\n",
    "        Y = []\n",
    "        for x in X:\n",
    "            _, y = self.viterbi(x)\n",
    "            Y.append(y)\n",
    "        return Y\n",
    "\n",
    "hmm = HMM(len(dataset.label2id), len(dataset.token2id))\n",
    "hmm.fit(train_X, train_Y)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "32868ed6",
   "metadata": {},
   "source": [
    "最后验证模型效果"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "4f4b7e88",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precision = 0.45762149610217284, recall = 0.3289222699093944, f1 = 0.38274259554692375\n",
      "precision = 0.4189636163175303, recall = 0.23270055113288426, f1 = 0.2992125984251969\n"
     ]
    }
   ],
   "source": [
    "def extract_entity(labels):\n",
    "    entity_list = []\n",
    "    entity_start = -1\n",
    "    entity_length = 0\n",
    "    entity_type = None\n",
    "    for token_index, label in enumerate(labels):\n",
    "        if label.startswith('B'):\n",
    "            if entity_start != -1:\n",
    "                # 遇到了一个新的B，将上一个实体加入列表\n",
    "                entity_list.append((entity_start, entity_length,\\\n",
    "                    entity_type))\n",
    "            # 记录新实体\n",
    "            entity_start = token_index\n",
    "            entity_length = 1\n",
    "            entity_type = label.split('-')[1]\n",
    "        elif label.startswith('I'):\n",
    "            if entity_start != -1:\n",
    "                # 上一个实体未关闭，遇到了一个新的I\n",
    "                if entity_type == label.split('-')[1]:\n",
    "                    # 若上一个实体与当前类型相同，长度+1\n",
    "                    entity_length += 1\n",
    "                else:\n",
    "                    # 若上一个实体与当前类型不同，\n",
    "                    # 将上一个实体加入列表，重置实体\n",
    "                    entity_list.append((entity_start, entity_length,\\\n",
    "                        entity_type))\n",
    "                    entity_start = -1\n",
    "                    entity_length = 0\n",
    "                    entity_type = None\n",
    "        else:\n",
    "            if entity_start != -1:\n",
    "                # 遇到了一个新的O，将上一个实体加入列表\n",
    "                entity_list.append((entity_start, entity_length,\\\n",
    "                    entity_type))\n",
    "            # 重置实体\n",
    "            entity_start = -1\n",
    "            entity_length = 0\n",
    "            entity_type = None\n",
    "    if entity_start != -1:\n",
    "        # 将上一个实体加入列表\n",
    "        entity_list.append((entity_start, entity_length, entity_type))\n",
    "    return entity_list\n",
    "\n",
    "def compute_metric(Y, P):\n",
    "    true_entity_set = set()\n",
    "    pred_entity_set = set()\n",
    "    for sent_no, labels in enumerate(Y):\n",
    "        labels = [dataset.id2label[x] for x in labels]\n",
    "        for ent in extract_entity(labels):\n",
    "            true_entity_set.add((sent_no, ent))\n",
    "    for sent_no, labels in enumerate(P):\n",
    "        labels = [dataset.id2label[x] for x in labels]\n",
    "        for ent in extract_entity(labels):\n",
    "            pred_entity_set.add((sent_no, ent))\n",
    "    if len(true_entity_set) > 0:\n",
    "        recall = len(true_entity_set & pred_entity_set)\\\n",
    "            / len(true_entity_set)\n",
    "    else:\n",
    "        recall = 0\n",
    "    if len(pred_entity_set) > 0:\n",
    "        precision = len(true_entity_set & pred_entity_set)\\\n",
    "            / len(pred_entity_set)\n",
    "    else:\n",
    "        precision = 0\n",
    "    if precision > 0 and recall > 0:\n",
    "        f1 = 2 * precision * recall / (precision + recall)\n",
    "    else:\n",
    "        f1 = 0\n",
    "    return precision, recall, f1\n",
    "\n",
    "train_P = hmm.decode(train_X)\n",
    "p, r, f = compute_metric(train_Y, train_P)\n",
    "print(f'precision = {p}, recall = {r}, f1 = {f}')\n",
    "test_P = hmm.decode(test_X)\n",
    "p, r, f = compute_metric(test_Y, test_P)\n",
    "print(f'precision = {p}, recall = {r}, f1 = {f}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3abe8397",
   "metadata": {},
   "source": [
    "\n",
    "\n",
    "下面展示基于条件随机场的序列标注模型。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "787f8ad1",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "代码修改自GitHub项目kmkurn/pytorch-crf\n",
    "（Copyright (c) 2019, Kemal Kurniawan, MIT License（见附录））\n",
    "\"\"\"\n",
    "import torch\n",
    "from torch import nn\n",
    "\n",
    "class CRFLayer(nn.Module):\n",
    "    def __init__(self, n_tags, n_features):\n",
    "        super().__init__()\n",
    "        self.n_tags = n_tags\n",
    "        self.n_features = n_features\n",
    "        # 定义模型参数\n",
    "        self.transitions = nn.Parameter(torch.empty(\\\n",
    "            n_tags, n_tags))\n",
    "        self.emission_weight = nn.Parameter(torch.empty(\\\n",
    "            n_features, n_tags))\n",
    "        self.start_transitions = nn.Parameter(torch.empty(n_tags))\n",
    "        self.end_transitions = nn.Parameter(torch.empty(n_tags))\n",
    "        self.reset_parameters()\n",
    "        \n",
    "    # 使用（-0.1,0.1）之间的均匀分布初始化参数\n",
    "    def reset_parameters(self):\n",
    "        nn.init.uniform_(self.transitions, -0.1, 0.1)\n",
    "        nn.init.uniform_(self.emission_weight, -0.1, 0.1)\n",
    "        nn.init.uniform_(self.start_transitions, -0.1, 0.1)\n",
    "        nn.init.uniform_(self.end_transitions, -0.1, 0.1)\n",
    "    \n",
    "    # 使用动态规划计算得分\n",
    "    def compute_score(self, emissions, tags, masks):\n",
    "        seq_len, batch_size, n_tags = emissions.shape\n",
    "        score = self.start_transitions[tags[0]] + \\\n",
    "            emissions[0, torch.arange(batch_size), tags[0]]\n",
    "        \n",
    "        for i in range(1, seq_len):\n",
    "            score += self.transitions[tags[i-1], tags[i]] * masks[i]\n",
    "            score += emissions[i, torch.arange(batch_size),\\\n",
    "                tags[i]] * masks[i]\n",
    "        \n",
    "        seq_ends = masks.long().sum(dim=0) - 1\n",
    "        last_tags = tags[seq_ends, torch.arange(batch_size)]\n",
    "        score += self.end_transitions[last_tags]\n",
    "        return score\n",
    "    \n",
    "    # 计算配分函数\n",
    "    def computer_normalizer(self, emissions, masks):\n",
    "        seq_len, batch_size, n_tags = emissions.shape\n",
    "        # batch_size * n_tags, [起始分数 + y_0为某标签的发射分数 ...]\n",
    "        score = self.start_transitions + emissions[0]\n",
    "        \n",
    "        for i in range(1, seq_len):\n",
    "            # batch_size * n_tags * 1 [y_{i-1}为某tag的总分]\n",
    "            broadcast_score = score.unsqueeze(2)\n",
    "            # batch_size * 1 * n_tags [y_i为某标签的发射分数]\n",
    "            broadcast_emissions = emissions[i].unsqueeze(1)\n",
    "            # batch_size * n_tags * n_tags [任意y_{i-1}到y_i的总分]\n",
    "            next_score = broadcast_score + self.transitions +\\\n",
    "                broadcast_emissions\n",
    "            # batch_size * n_tags [对y_{i-1}求和]\n",
    "            next_score = torch.logsumexp(next_score, dim=1)\n",
    "            # masks为True则更新，否则保留\n",
    "            score = torch.where(masks[i].unsqueeze(1), next_score, score)\n",
    "            \n",
    "        score += self.end_transitions\n",
    "        return torch.logsumexp(score, dim=1)\n",
    "    \n",
    "    def forward(self, features, tags, masks):\n",
    "        \"\"\"\n",
    "        features: seq_len * batch_size * n_features\n",
    "        tags/masks: seq_len * batch_size\n",
    "        \"\"\"\n",
    "        _, batch_size, _ = features.size()\n",
    "        emissions = torch.matmul(features, self.emission_weight)\n",
    "        masks = masks.to(torch.bool)\n",
    "        \n",
    "        score = self.compute_score(emissions, tags, masks)\n",
    "        partition = self.computer_normalizer(emissions, masks)\n",
    "        \n",
    "        likelihood = score - partition\n",
    "        return likelihood.sum() / batch_size\n",
    "    \n",
    "    def decode(self, features, masks):\n",
    "        # 与computer_normalizer类似，sum变为max\n",
    "        emissions = torch.matmul(features, self.emission_weight)\n",
    "        masks = masks.to(torch.bool)\n",
    "        \n",
    "        seq_len, batch_size, n_tags = emissions.shape\n",
    "        score = self.start_transitions + emissions[0]\n",
    "        history = []\n",
    "        \n",
    "        for i in range(1, seq_len):\n",
    "            broadcast_score = score.unsqueeze(2)\n",
    "            broadcast_emission = emissions[i].unsqueeze(1)\n",
    "            \n",
    "            next_score = broadcast_score + self.transitions +\\\n",
    "                broadcast_emission\n",
    "            next_score, indices = next_score.max(dim=1)\n",
    "            \n",
    "            score = torch.where(masks[i].unsqueeze(1), next_score,\\\n",
    "                score)\n",
    "            history.append(indices)\n",
    "            \n",
    "        score += self.end_transitions\n",
    "        seq_ends = masks.long().sum(dim=0) - 1\n",
    "        best_tags_list = []\n",
    "        \n",
    "        for idx in range(batch_size):\n",
    "            _, best_last_tag = score[idx].max(dim=0)\n",
    "            best_tags = [best_last_tag.item()]\n",
    "            \n",
    "            for hist in reversed(history[:seq_ends[idx]]):\n",
    "                best_last_tag = hist[idx][best_tags[-1]]\n",
    "                best_tags.append(best_last_tag.item())\n",
    "                \n",
    "            best_tags.reverse()\n",
    "            best_tags_list.append(best_tags)\n",
    "            \n",
    "        return best_tags_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "69f4af96",
   "metadata": {},
   "outputs": [],
   "source": [
    "class CRF(nn.Module):\n",
    "    def __init__(self, vocab_size, hidden_size, n_tags):\n",
    "        super().__init__()\n",
    "        self.embedding = nn.Embedding(vocab_size, hidden_size)\n",
    "        self.crf = CRFLayer(n_tags, hidden_size)\n",
    "        \n",
    "    def forward(self, input_ids, masks, labels):\n",
    "        \"\"\"\n",
    "        input_ids/masks/labels: batch_size * seq_len\n",
    "        \"\"\"\n",
    "        # 将输入序列转化为词嵌入序列\n",
    "        # batch_size * seq_len * embed_size\n",
    "        embed = self.embedding(input_ids)\n",
    "        embed = torch.transpose(embed, 0, 1)\n",
    "        masks = torch.transpose(masks, 0, 1)\n",
    "        labels = torch.transpose(labels, 0, 1)\n",
    "        # 隐状态和标签、掩码输入条件随机场计算损失\n",
    "        llh = self.crf(embed, labels, masks)\n",
    "        return -llh\n",
    "    \n",
    "    def decode(self, input_ids, masks):\n",
    "        embed = self.embedding(input_ids)\n",
    "        embed = torch.transpose(embed, 0, 1)\n",
    "        masks = torch.transpose(masks, 0, 1)\n",
    "        # 调用CRFLayer进行解码\n",
    "        return self.crf.decode(embed, masks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "178948b4",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "epoch-19, loss=50.41: 100%|█| 20/20 [05:24<00:00, 16.21s/it]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkQAAAGwCAYAAABIC3rIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABY3ElEQVR4nO3dd3iUVeL28e8kk0z6pJGEQGgSahAQEAEVlKYIKOqioKi7rhXRKCD6c13LKiiroMJa11dcG64Krl1AJYBIkSY9lEBCCaGkJ2SSmef9I2R0qCFMMjPJ/bmuuS7mec48cw5xyb2nmgzDMBARERFpwPw8XQERERERT1MgEhERkQZPgUhEREQaPAUiERERafAUiERERKTBUyASERGRBk+BSERERBo8s6cr4CscDgf79u0jPDwck8nk6eqIiIhINRiGQWFhIYmJifj5nbofSIGomvbt20dSUpKnqyEiIiI1kJWVRdOmTU95X4GomsLDw4HKv9CIiAgP10ZERESqo6CggKSkJOfv8VNRIKqmqmGyiIgIBSIREREfc6bpLppULSIiIg2eApGIiIg0eApEIiIi0uBpDpGIiAhgt9spLy/3dDXkLAUEBODv73/Oz1EgEhGRBs0wDLKzs8nLy/N0VaSGIiMjSUhIOKd9AhWIRESkQasKQ3FxcYSEhGjzXR9iGAYlJSXk5OQA0Lhx4xo/S4FIREQaLLvd7gxDMTExnq6O1EBwcDAAOTk5xMXF1Xj4TJOqRUSkwaqaMxQSEuLhmsi5qPr5ncscMAUiERFp8DRM5tvc8fNTIBIREZEGT4FIREREGjwFIhEREaFFixa89NJLHn+Gp2iVmYeVVdjJOlJCXEQQEUEBnq6OiIj4iH79+tGlSxe3BZCVK1cSGhrqlmf5IvUQedjIN5YxYNoiftlx2NNVERGResYwDCoqKqpVtlGjRg16tZ0CkYc1j678j2/XoWIP10RERODYZn+2Co+8DMOoVh1vu+020tLSePnllzGZTJhMJnbt2sXChQsxmUx8//33dO/eHYvFwuLFi9mxYwdXX3018fHxhIWF0aNHDxYsWODyzOOHu0wmE//+978ZMWIEISEhJCcn88UXX5zV32VmZiZXX301YWFhREREMHLkSA4cOOC8v27dOi677DLCw8OJiIigW7du/PrrrwDs3r2bYcOGERUVRWhoKB07duSbb745q+8/Gxoy87AWMccC0eESD9dEREQASsvtdPj79x757k1PDyYk8My/ml9++WXS09NJSUnh6aefBip7eHbt2gXAww8/zAsvvECrVq2IjIxkz549DBkyhGeeeYagoCDeffddhg0bxtatW2nWrNkpv+epp55i6tSp/POf/2TGjBncdNNN7N69m+jo6DPW0TAMrrnmGkJDQ0lLS6OiooJ7772XG264gYULFwJw00030bVrV1577TX8/f1Zu3YtAQGV00fGjh2LzWZj0aJFhIaGsmnTJsLCws74vTXl0R6iRYsWMWzYMBITEzGZTHz++efOe+Xl5UyaNIlOnToRGhpKYmIit9xyC/v27XN5RllZGePGjSM2NpbQ0FCGDx/Onj17XMrk5uYyZswYrFYrVquVMWPGeM2ZNc1jKsdr1UMkIiLVZbVaCQwMJCQkhISEBBISElx2aH766acZOHAg5513HjExMXTu3Jm77rqLTp06kZyczDPPPEOrVq3O2ONz2223MWrUKFq3bs3kyZMpLi5mxYoV1arjggUL+O233/jwww/p1q0bPXv25L333iMtLY2VK1cClT1IAwYMoF27diQnJ/OnP/2Jzp07O+/16dOHTp060apVK4YOHcqll15aw7+xM/NoD1FxcTGdO3fmz3/+M9ddd53LvZKSElavXs3jjz9O586dyc3NJTU1leHDhzu70wBSU1P58ssvmT17NjExMYwfP56hQ4eyatUq538co0ePZs+ePXz33XcA3HnnnYwZM4Yvv/yy7hp7Ci1iKwPR7sMKRCIi3iA4wJ9NTw/22He7Q/fu3V3eFxcX89RTT/HVV1+xb98+KioqKC0tJTMz87TPOf/8851/Dg0NJTw83Hlu2Jls3ryZpKQkkpKSnNc6dOhAZGQkmzdvpkePHjz00EP89a9/5b333mPAgAH86U9/4rzzzgPg/vvv55577mHevHkMGDCA6667zqU+7ubRQHTllVdy5ZVXnvSe1Wpl/vz5LtdmzJjBhRdeSGZmJs2aNSM/P5+3337b+RcJ8P7775OUlMSCBQsYPHgwmzdv5rvvvmPZsmX07NkTgLfeeotevXqxdetW2rZte9LvLysro6yszPm+oKDAHU0+QdWQ2b78oxwttxPkpv8xiIhIzZhMpmoNW3mz41eLTZw4ke+//54XXniB1q1bExwczPXXX4/NZjvtc6qGr6qYTCYcDke16mAYxkl3kP7j9SeffJLRo0fz9ddf8+233/LEE08we/ZsRowYwV//+lcGDx7M119/zbx585gyZQovvvgi48aNq9b3ny2fmlSdn5+PyWQiMjISgFWrVlFeXs6gQYOcZRITE0lJSWHp0qUA/PLLL1itVmcYArjooouwWq3OMiczZcoU5xCb1Wp1SbjuFB0aSHhQ5f/wMo9oHpGIiFRPYGAgdru9WmUXL17MbbfdxogRI+jUqRMJCQnO+Ua1pUOHDmRmZpKVleW8tmnTJvLz82nfvr3zWps2bXjwwQeZN28e1157Le+8847zXlJSEnfffTdz5sxh/PjxvPXWW7VWX58JREePHuWRRx5h9OjRREREAJCdnU1gYCBRUVEuZePj48nOznaWiYuLO+F5cXFxzjIn8+ijj5Kfn+98/fEH6k4mk4kWx+YRZWgekYiIVFOLFi1Yvnw5u3bt4tChQ6ftuWndujVz5sxh7dq1rFu3jtGjR1e7p6emBgwYwPnnn89NN93E6tWrWbFiBbfccgt9+/ale/fulJaWct9997Fw4UJ2797Nzz//zMqVK51hKTU1le+//56MjAxWr17Njz/+6BKk3M0nAlF5eTk33ngjDoeDV1999Yzlj++mO1OX3clYLBYiIiJcXrVF84hERORsTZgwAX9/fzp06ECjRo1OOx9o+vTpREVF0bt3b4YNG8bgwYO54IILarV+VYuloqKiuPTSSxkwYACtWrXi448/BsDf35/Dhw9zyy230KZNG0aOHMmVV17JU089BYDdbmfs2LG0b9+eK664grZt21YrA9SU1w+SlpeXM3LkSDIyMvjxxx9dgklCQgI2m43c3FyXXqKcnBx69+7tLPPHPQ+qHDx4kPj4+NpvQDVUzSPKOKQhMxERqZ42bdrwyy+/uFxr0aLFSfcyatGiBT/++KPLtbFjx7q8P34I7WTPOdMK7eOf0axZM/73v/+dtGxgYCAfffTRKZ81Y8aM036Xu3l1D1FVGNq2bRsLFiwgJibG5X63bt0ICAhwmXy9f/9+NmzY4AxEvXr1Ij8/32WZ4PLly8nPz3eW8bSqITP1EImIiHiGR3uIioqK2L59u/N9RkYGa9euJTo6msTERK6//npWr17NV199hd1ud875iY6OJjAwEKvVyu2338748eOJiYkhOjqaCRMm0KlTJ+eqs6qutjvuuIM33ngDqFx2P3To0FOuMKtrLWK1W7WIiIgneTQQ/frrr1x22WXO9w899BAAt956K08++aRzw6guXbq4fO6nn36iX79+QOW4qNlsZuTIkZSWltK/f39mzZrlskHVBx98wP333+9cjTZ8+HBmzpxZiy07O1U9RFp6LyIi4hkeDUT9+vU77bkt1TnTJSgoiBkzZpx2rDE6Opr333+/RnWsC9GhgYRbzBSWVZB1pITk+HBPV0lEpEGp7hli4p3c8fPz6jlEDYXJZKJ5bNXEag2biYjUlaqNB0tKtKjFl1X9/I7fSPJseP0qs4aiRUwoG/YWsFuHvIqI1Bl/f38iIyOdx1GEhIScdksW8S6GYVBSUkJOTg6RkZEu02XOlgKRl3BuzqiVZiIidSohIQGg2md0ifeJjIx0/hxrSoHIS2hzRhERzzCZTDRu3Ji4uDjKy8s9XR05SwEBAefUM1RFgchLVG3OuEubM4qIeIS/v79bfrGKb9Kkai9R1UO0L7+Uo+XVO6xPRERE3EOByEvEhAYSZjFjGJClU+9FRETqlAKRlzCZTM4dq7X0XkREpG4pEHmRJpHBAGQXHPVwTURERBoWBSIvEhceBEBOQZmHayIiItKwKBB5kbhwCwA5heohEhERqUsKRF4kPuJYD1GheohERETqkgKRF2kUUdlDdEBDZiIiInVKgciLVA2ZHdSQmYiISJ1SIPIiVUNmh4ttlNsdHq6NiIhIw6FA5EWiQwIx+5kwDDhUpGEzERGRuqJA5EX8/EzEhh1baaZ5RCIiInVGgcjLxEdULb1XIBIREakrCkReptGxzRkPaLdqERGROqNA5GXi1EMkIiJS5xSIvEz8sR4iLb0XERGpOwpEXiZOmzOKiIjUOQUiL6PzzEREROqeApGXcZ5nph4iERGROqNA5GWqeogOFZVhdxgero2IiEjDoEDkZWLCLPiZwGHAYe1WLSIiUicUiLyM/x93q9bSexERkTqhQOSFfl9pponVIiIidUGByAvFHduLSD1EIiIidUOByAs5l95rpZmIiEidUCDyQnHHlt4f0F5EIiIidUKByAuph0hERKRuKRB5oapApPPMRERE6oYCkReq2q1a55mJiIjUDQUiL1S17P5QURkO7VYtIiJS6xSIvFBsmAWTCSocBkdKbJ6ujoiISL2nQOSFAvz9iAkNBDSxWkREpC4oEHmpRuFaei8iIlJXFIi81O8rzdRDJCIiUtsUiLxU1QGvh3TivYiISK1TIPJSjdRDJCIiUmcUiLxUbFjlpOpDRVplJiIiUtsUiLxUI+1WLSIiUmcUiLxUI+ccIvUQiYiI1DYFIi8VqzlEIiIidUaByEtV9RDll5Zjq3B4uDYiIiL1m0cD0aJFixg2bBiJiYmYTCY+//xzl/uGYfDkk0+SmJhIcHAw/fr1Y+PGjS5lysrKGDduHLGxsYSGhjJ8+HD27NnjUiY3N5cxY8ZgtVqxWq2MGTOGvLy8Wm7dubEGB2D2MwFwuFi9RCIiIrXJo4GouLiYzp07M3PmzJPenzp1KtOmTWPmzJmsXLmShIQEBg4cSGFhobNMamoqc+fOZfbs2SxZsoSioiKGDh2K3W53lhk9ejRr167lu+++47vvvmPt2rWMGTOm1tt3Lvz8TMQcW2mmYTMREZFaZngJwJg7d67zvcPhMBISEoznnnvOee3o0aOG1Wo1Xn/9dcMwDCMvL88ICAgwZs+e7Syzd+9ew8/Pz/juu+8MwzCMTZs2GYCxbNkyZ5lffvnFAIwtW7acsj5Hjx418vPzna+srCwDMPLz893V5DO66pVFRvNJXxk/bM6us+8UERGpT/Lz86v1+9tr5xBlZGSQnZ3NoEGDnNcsFgt9+/Zl6dKlAKxatYry8nKXMomJiaSkpDjL/PLLL1itVnr27Oksc9FFF2G1Wp1lTmbKlCnOITar1UpSUpK7m3hGVbtVq4dIRESkdnltIMrOzgYgPj7e5Xp8fLzzXnZ2NoGBgURFRZ22TFxc3AnPj4uLc5Y5mUcffZT8/HznKysr65zaUxNaei8iIlI3zJ6uwJmYTCaX94ZhnHDteMeXOVn5Mz3HYrFgsVjOsrbupaX3IiIidcNre4gSEhIATujFycnJcfYaJSQkYLPZyM3NPW2ZAwcOnPD8gwcPntD75G2cQ2Y64FVERKRWeW0gatmyJQkJCcyfP995zWazkZaWRu/evQHo1q0bAQEBLmX279/Phg0bnGV69epFfn4+K1ascJZZvnw5+fn5zjLequr4jkPqIRIREalVHh0yKyoqYvv27c73GRkZrF27lujoaJo1a0ZqaiqTJ08mOTmZ5ORkJk+eTEhICKNHjwbAarVy++23M378eGJiYoiOjmbChAl06tSJAQMGANC+fXuuuOIK7rjjDt544w0A7rzzToYOHUrbtm3rvtFnoeqAV/UQiYiI1C6PBqJff/2Vyy67zPn+oYceAuDWW29l1qxZPPzww5SWlnLvvfeSm5tLz549mTdvHuHh4c7PTJ8+HbPZzMiRIyktLaV///7MmjULf39/Z5kPPviA+++/37kabfjw4afc+8ibOCdVq4dIRESkVpkMwzA8XQlfUFBQgNVqJT8/n4iIiDr5zrwSG12erhwO3PrMFVjM/mf4hIiIiPxRdX9/e+0cIqk8viPAv3IlnJbei4iI1B4FIi9mMpmcK800bCYiIlJ7FIi8nHarFhERqX0KRF6uaqXZIa00ExERqTUKRF7OuReRApGIiEitUSDychoyExERqX0KRF7u9x4irTITERGpLQpEXk49RCIiIrVPgcjLOZfdaw6RiIhIrVEg8nJVQ2bqIRIREak9CkReruo8s8KyCo6W2z1cGxERkfpJgcjLRQSbCfSv/DFp2ExERKR2KBB5ucrjOyo3Z9SwmYiISO1QIPIBsVp6LyIiUqsUiHxATGhlD9FhDZmJiIjUCgUiHxAdWtlDdLhYPUQiIiK1QYHIB1TNITqiQCQiIlIrFIh8QHSoApGIiEhtUiDyAVWBSMvuRUREaocCkQ+I0ZCZiIhIrVIg8gFVk6oViERERGqHApEPcC67L7ZhGIaHayMiIlL/KBD5gKohM1uFg2KbzjMTERFxNwUiHxASaCYooPJHpc0ZRURE3E+ByEfEaHNGERGRWqNA5COcK810npmIiIjbKRD5CG3OKCIiUnsUiHyEc3PGYs0hEhERcTcFIh9RtfReQ2YiIiLup0DkI7Q5o4iISO1RIPIRVZOqtcpMRETE/RSIfESMJlWLiIjUGgUiH1E1qVobM4qIiLifApGP+OPGjDrPTERExL0UiHxE1RyisgoHJTrPTERExK0UiHxESKA/FnPlj0vziERERNxLgchHmEwm58RqrTQTERFxLwUiHxIdponVIiIitUGByIfoxHsREZHaoUDkQ7QXkYiISO1QIPIhOvFeRESkdigQ+ZDf5xApEImIiLiTApEP+X2VmSZVi4iIuJMCkQ+J0Yn3IiIitUKByIdoyExERKR2KBD5EK0yExERqR1eHYgqKir429/+RsuWLQkODqZVq1Y8/fTTOBwOZxnDMHjyySdJTEwkODiYfv36sXHjRpfnlJWVMW7cOGJjYwkNDWX48OHs2bOnrptzzqpWmZWW2ymxVXi4NiIiIvWHVwei559/ntdff52ZM2eyefNmpk6dyj//+U9mzJjhLDN16lSmTZvGzJkzWblyJQkJCQwcOJDCwkJnmdTUVObOncvs2bNZsmQJRUVFDB06FLvdtw5JDbOYCTx2npmGzURERNzHqwPRL7/8wtVXX81VV11FixYtuP766xk0aBC//vorUNk79NJLL/HYY49x7bXXkpKSwrvvvktJSQkffvghAPn5+bz99tu8+OKLDBgwgK5du/L++++zfv16FixY4MnmnbU/nmemYTMRERH38epAdPHFF/PDDz+Qnp4OwLp161iyZAlDhgwBICMjg+zsbAYNGuT8jMVioW/fvixduhSAVatWUV5e7lImMTGRlJQUZ5mTKSsro6CgwOXlDaK19F5ERMTtzJ6uwOlMmjSJ/Px82rVrh7+/P3a7nWeffZZRo0YBkJ2dDUB8fLzL5+Lj49m9e7ezTGBgIFFRUSeUqfr8yUyZMoWnnnrKnc1xi4SIIDbuK2B//lFPV0VERKTe8Ooeoo8//pj333+fDz/8kNWrV/Puu+/ywgsv8O6777qUM5lMLu8Nwzjh2vHOVObRRx8lPz/f+crKyqp5Q9yoSVQwAHtzSz1cExERkfrDq3uIJk6cyCOPPMKNN94IQKdOndi9ezdTpkzh1ltvJSEhAajsBWrcuLHzczk5Oc5eo4SEBGw2G7m5uS69RDk5OfTu3fuU322xWLBYLLXRrHPSJLIyEO3LUyASERFxF6/uISopKcHPz7WK/v7+zmX3LVu2JCEhgfnz5zvv22w20tLSnGGnW7duBAQEuJTZv38/GzZsOG0g8laJxwLRXgUiERERt/HqHqJhw4bx7LPP0qxZMzp27MiaNWuYNm0af/nLX4DKobLU1FQmT55McnIyycnJTJ48mZCQEEaPHg2A1Wrl9ttvZ/z48cTExBAdHc2ECRPo1KkTAwYM8GTzakRDZiIiIu7n1YFoxowZPP7449x7773k5OSQmJjIXXfdxd///ndnmYcffpjS0lLuvfdecnNz6dmzJ/PmzSM8PNxZZvr06ZjNZkaOHElpaSn9+/dn1qxZ+Pv7e6JZ56TpsR6i7IKjVNgdmP29upNPRETEJ5gMwzA8XQlfUFBQgNVqJT8/n4iICI/Vw+EwaPf4d9jsDpZMuoymUSEeq4uIiIi3q+7vb3Uv+Bg/PxONI4MADZuJiIi4iwKRD3KuNMtXIBIREXEHBSIf5Fxpph4iERERt1Ag8kFNtPReRETErRSIfJBz6X2eju8QERFxBwUiH9TUOWRW4uGaiIiI1A8KRD7oj7tVa9cEERGRc6dA5IOqlt0fLXeQW1Lu4dqIiIj4PgUiH2Qx+xMXXnnwrFaaiYiInDsFIh/1+7CZ5hGJiIicKwUiH6WVZiIiIu6jQOSjmmpzRhEREbdRIPJRv/cQachMRETkXCkQ+ahE67HzzDRkJiIics4UiHzU7z1EGjITERE5VwpEPqoqEB0ptlFiq/BwbURERHybApGPiggKINxiBjRsJiIicq4UiHxYVS/RPg2biYiInBMFIh8WG1a5W/WRYpuHayIiIuLbFIh8WFRoIKBAJCIicq4UiHxYVEgAALklCkQiIiLnQoHIh0WFVPYQKRCJiIicGwUiH+bsISou93BNREREfJsCkQ+rmkOkHiIREZFzo0Dkw6I1qVpERMQtFIh8mOYQiYiIuIcCkQ/7fcisHMMwPFwbERER36VA5MOqJlXbKhyU2Owero2IiIjvUiDyYcEB/ljMlT9CDZuJiIjUnAKRDzOZTL/PI9LSexERkRpTIPJxWnovIiJy7hSIfFx0qI7vEBEROVc1CkTvvvsuX3/9tfP9ww8/TGRkJL1792b37t1uq5ycWWSI9iISERE5VzUKRJMnTyY4OBiAX375hZkzZzJ16lRiY2N58MEH3VpBOb3okN+X3ouIiEjNmGvyoaysLFq3bg3A559/zvXXX8+dd95Jnz596NevnzvrJ2fw+3lm6iESERGpqRr1EIWFhXH48GEA5s2bx4ABAwAICgqitLTUfbWTM9KkahERkXNXox6igQMH8te//pWuXbuSnp7OVVddBcDGjRtp0aKFO+snZxCtQCQiInLOatRD9K9//YtevXpx8OBBPvvsM2JiYgBYtWoVo0aNcmsF5fR+n1StOUQiIiI1VaMeosjISGbOnHnC9aeeeuqcKyRnp2pSdZ56iERERGqsRj1E3333HUuWLHG+/9e//kWXLl0YPXo0ubm5bqucnFnksUnVWnYvIiJSczUKRBMnTqSgoACA9evXM378eIYMGcLOnTt56KGH3FpBOb2qOURlFQ5KdcCriIhIjdRoyCwjI4MOHToA8NlnnzF06FAmT57M6tWrGTJkiFsrKKcXEuhPoL8fNruDIyU2mgQGe7pKIiIiPqdGPUSBgYGUlJQAsGDBAgYNGgRAdHS0s+dI6obJZCIqVHsRiYiInIsa9RBdfPHFPPTQQ/Tp04cVK1bw8ccfA5Cenk7Tpk3dWkE5s6iQQA4UlGnpvYiISA3VqIdo5syZmM1mPv30U1577TWaNGkCwLfffssVV1zh1grKmUXpPDMREZFzUqMeombNmvHVV1+dcH369OnnXCE5e1UTq/N0npmIiEiN1KiHCMBut/PZZ5/xzDPP8OyzzzJnzhzsdvevctq7dy8333wzMTExhISE0KVLF1atWuW8bxgGTz75JImJiQQHB9OvXz82btzo8oyysjLGjRtHbGwsoaGhDB8+nD179ri9rp6ipfciIiLnpkaBaPv27bRv355bbrmFOXPm8OmnnzJmzBg6duzIjh073Fa53Nxc+vTpQ0BAAN9++y2bNm3ixRdfJDIy0llm6tSpTJs2jZkzZ7Jy5UoSEhIYOHAghYWFzjKpqanMnTuX2bNns2TJEoqKihg6dGitBDhP+L2HSIFIRESkJkyGYRhn+6EhQ4ZgGAYffPAB0dHRABw+fJibb74ZPz8/vv76a7dU7pFHHuHnn39m8eLFJ71vGAaJiYmkpqYyadIkoLI3KD4+nueff5677rqL/Px8GjVqxHvvvccNN9wAwL59+0hKSuKbb75h8ODB1apLQUEBVquV/Px8IiIi3NI+d/l/SzJ4+qtNDOucyIxRXT1dHREREa9R3d/fNeohSktLY+rUqc4wBBATE8Nzzz1HWlpaTR55Ul988QXdu3fnT3/6E3FxcXTt2pW33nrLeT8jI4Ps7Gznsn8Ai8VC3759Wbp0KVB5vlp5eblLmcTERFJSUpxlTqasrIyCggKXl7fSsnsREZFzU6NAZLFYXIakqhQVFREYGHjOlaqyc+dOXnvtNZKTk/n++++5++67uf/++/nPf/4DQHZ2NgDx8fEun4uPj3fey87OJjAwkKioqFOWOZkpU6ZgtVqdr6SkJLe1y92qVplp2b2IiEjN1CgQDR06lDvvvJPly5djGAaGYbBs2TLuvvtuhg8f7rbKORwOLrjgAiZPnkzXrl256667uOOOO3jttddcyplMJpf3hmGccO14Zyrz6KOPkp+f73xlZWXVvCG1zBmI1EMkIiJSIzUKRK+88grnnXcevXr1IigoiKCgIHr37k3r1q156aWX3Fa5xo0bO48IqdK+fXsyMzMBSEhIADihpycnJ8fZa5SQkIDNZjvh0Nk/ljkZi8VCRESEy8tbVU2qztWyexERkRqpUSCKjIzkf//7H+np6Xz66ad88sknpKenM3fuXJcVYOeqT58+bN261eVaeno6zZs3B6Bly5YkJCQwf/58532bzUZaWhq9e/cGoFu3bgQEBLiU2b9/Pxs2bHCW8XVVy+5Ly+064FVERKQGqr0x45lOsV+4cKHzz9OmTatxhf7owQcfpHfv3kyePJmRI0eyYsUK3nzzTd58802gcqgsNTWVyZMnk5ycTHJyMpMnTyYkJITRo0cDYLVauf322xk/fjwxMTFER0czYcIEOnXqxIABA9xST08Ls5gJ8DdRbjfILbERrANeRUREzkq1A9GaNWuqVe5Mc3fORo8ePZg7dy6PPvooTz/9NC1btuSll17ipptucpZ5+OGHKS0t5d577yU3N5eePXsyb948wsPDnWWmT5+O2Wxm5MiRlJaW0r9/f2bNmoW/v7/b6upJJpOJqJBAcgorzzNLjFQgEhERORs12oeoIfLmfYgArnhpEVuyC/nPXy7k0jaNPF0dERERr1Cr+xCJ92kWHQLA9pwiD9dERETE9ygQ1RPtEiqHCLdmn7g/lIiIiJyeAlE90TahshtwywEFIhERkbOlQFRPtD3WQ5SeXYjDoWlhIiIiZ0OBqJ5oEROCxexHabmdzCMlnq6OiIiIT1EgqifM/n4kx4cBsEXziERERM6KAlE90ja+ch6RJlaLiIicHQWiesS50uxAgYdrIiIi4lsUiOqRdo0rA9GW/eohEhERORsKRPVI1UqzXYeLOVquQ15FRESqS4GoHmkUZiE6NBCHAdsOaMdqERGR6lIgqkdMJhNt4yt7iTZnax6RiIhIdSkQ1TNtdYSHiIjIWVMgqmfaN1YgEhEROVsKRPWM80wzBSIREZFqUyCqZ9rEh2EywaGiMg4VlXm6OiIiIj5BgaieCQk00zw6BICN+zSxWkREpDoUiOqhC5pFAbAi47CHayIiIuIbFIjqoYtaxQCwbOcRD9dERETENygQ1UO9zqsMROuy8iixVXi4NiIiIt5PgageahoVTJPIYCocBqt253q6OiIiIl5PgageMplM9GwVDcCynZpHJCIiciYKRPWU5hGJiIhUnwJRPdWr1e/ziIrLNI9IRETkdBSI6qmk6BDNIxIREakmBaJ67PdhM80jEhEROR0FonrsIk2sFhERqRYFonqsqofotz35mkckIiJyGgpE9dgf5xGtyczzdHVERES8lgJRPdclKRKADfvyPVsRERERL6ZAVM91SIwAdPK9iIjI6SgQ1XMpTawAbNyrHiIREZFTUSCq5zoe6yHaeaiYIk2sFhEROSkFonouNsxCQkQQAJv3a9hMRETkZBSIGoCUJpW9RBs0bCYiInJSCkQNQMfEynlEG/aqh0hERORkFIgagI7OlWbqIRIRETkZBaIGoGql2bacIo6W2z1cGxEREe+jQNQANLYGER0aiN1hsDW70NPVERER8ToKRA2AyWT6w7CZ5hGJiIgcT4GogXBOrNY8IhERkRMoEDUQzh4iLb0XERE5gQJRA1E1sXpzdiHldoeHayMiIuJdFIgaiObRIYRZzNgqHGw7UOTp6oiIiHgVBaIGws/PRNdmkQD8svOwZysjIiLiZRSIGpBLkxsBkJZ+0MM1ERER8S4+FYimTJmCyWQiNTXVec0wDJ588kkSExMJDg6mX79+bNy40eVzZWVljBs3jtjYWEJDQxk+fDh79uyp49p73qVtKgPR8p2HtUGjiIjIH/hMIFq5ciVvvvkm559/vsv1qVOnMm3aNGbOnMnKlStJSEhg4MCBFBb+vgFhamoqc+fOZfbs2SxZsoSioiKGDh2K3d6wQkGb+DASIoIoq3CwPOOIp6sjIiLiNXwiEBUVFXHTTTfx1ltvERUV5bxuGAYvvfQSjz32GNdeey0pKSm8++67lJSU8OGHHwKQn5/P22+/zYsvvsiAAQPo2rUr77//PuvXr2fBggWeapJHmEwmLm0TC8AiDZuJiIg4+UQgGjt2LFdddRUDBgxwuZ6RkUF2djaDBg1yXrNYLPTt25elS5cCsGrVKsrLy13KJCYmkpKS4ixzMmVlZRQUFLi86oO+beIABSIREZE/8vpANHv2bFavXs2UKVNOuJednQ1AfHy8y/X4+HjnvezsbAIDA116lo4vczJTpkzBarU6X0lJSefaFK9wcetY/EyVB73uyyv1dHVERES8glcHoqysLB544AHef/99goKCTlnOZDK5vDcM44RrxztTmUcffZT8/HznKysr6+wq76WsIQF0SYoE1EskIiJSxasD0apVq8jJyaFbt26YzWbMZjNpaWm88sormM1mZ8/Q8T09OTk5znsJCQnYbDZyc3NPWeZkLBYLERERLq/6omq1mZbfi4iIVPLqQNS/f3/Wr1/P2rVrna/u3btz0003sXbtWlq1akVCQgLz5893fsZms5GWlkbv3r0B6NatGwEBAS5l9u/fz4YNG5xlGpqqQLRk+yEqdIyHiIgIZk9X4HTCw8NJSUlxuRYaGkpMTIzzempqKpMnTyY5OZnk5GQmT55MSEgIo0ePBsBqtXL77bczfvx4YmJiiI6OZsKECXTq1OmESdoNReemkUSGBJBXUs7KXbn0Oi/G01USERHxKK8ORNXx8MMPU1payr333ktubi49e/Zk3rx5hIeHO8tMnz4ds9nMyJEjKS0tpX///syaNQt/f38P1txz/P1MDO6QwMe/ZjFn9R4FIhERafBMhmEYnq6ELygoKMBqtZKfn18v5hOtyDjCyDd+ITTQn5V/G0BIoM9nYxERkRNU9/e3V88hktrTo0UUzaJDKLbZ+X7jqbcfEBERaQgUiBook8nEdRc0BeCzVXs9XBsRERHPUiBqwK69oAkAP+84pE0aRUSkQVMgasCSokPo2TIaw4C5a07sJbI7DIrLKjxQMxERkbqlQNTAXdetathsDw7H7/PrN+zN5/IXF3LRlB84XFTmqeqJiIjUCQWiBm5Ip8YEB/iz81AxV768mK9/2897y3Zz7atL2X24hMKjFazanXvmB4mIiPgwBaIGLsxi5tkRKYRbzGw9UMjYD1fz+OcbsNkdBAdU7tO0NbvQw7UUERGpXQpEwrUXNGXJpMt5oH8y4RYz/n4m/m9IO1IHJAOwRYFIRETqOe3GJwBYQwJ4cGAb7ri0FcVlFcRHBLHo2OGvm7MLPFw7ERGR2qVAJC7CLGbCLJX/WbRrXHn8ya5DxZTa7AQHNsyjTkREpP7TkJmcUqMwCzGhgTgM2JajYTMREam/FIjklEwmk7OXSPOIRESkPlMgktNqG195EN6W/QpEIiJSfykQyWn93kOkidUiIlJ/KRDJabVPONZDlF2IYRhnKC0iIuKbFIjktJLjw/AzwZFiGwfPcITH5v0F2sRRRER8kpbdy2kFBfjTMjaUHQeL2bK/kLjwIJf7tgoH327Yz6ylu1iTmUdQgB+LHr7shHIiIiLeTD1EckbtnMNmJ84juuM/v/LA7LWsycwD4Gi5g9U6+0xERHyMApGcUbuEYxOrj1tplnWkhLT0g/j7mXhwQBuu6tQYgLVZ+XVeRxERkXOhQCRn1K5xZQ/R5uPmBy3dcQiAzk2tPDAgmUuSYwH4bU9endZPRETkXCkQyRlV9RDtyCmi3O5wXv95+2EA+rSuDEKdkyIBWL8nH4dDK9JERMR3KBDJGTWNCibcYsZmd/DbnsrhMMMwWLrDNRAlx4URFOBHYVkFOw8Ve6y+IiIiZ0uBSM7IZDJxefs4AD5dtQeA9ANFHCoqIyjAj67NIgEw+/vRqYkVgHVZeZ6oqoiISI0oEEm13NA9CYAv1+2jxFbBku2V84cubBmDxezvLNe5aSQA6zSPSEREfIgCkVTLRa1iaBYdQlFZBd+sz2bpsUDU57wYl3LnH5tHtG6PVpqJiIjvUCCSavHzMzGye1MAPly+m+UZR4Df5w9V6XKsh2jzvgJsFQ5ERER8gQKRVNv13ZLwM8HqzDyKyiqIDAmgw7El+VWSooOJCgnAZnfoQFgREfEZCkRSbQnWIPq2aeR83/u8GPz8TC5lTCYT51fNI9LEahER8REKRHJWbuiR5Pxz7/NiT1qms+YRiYiIj1EgkrNyebt4Eq1BBPr7ufQW/VHnplp6LyIivkWn3ctZCTT78ck9vSkoLScpOuSkZaqGzLYfLCK32EZUaGAd1lBEROTsqYdIzlqTyGDaHzeZ+o8ahVvo0DgCw4D3lu2uw5qJiIjUjAKR1Iq7+50HwDs/Z1Biq/BwbURERE5PgUhqxVWdGtMiJoTcknI+WpHl6eqIiIiclgKR1Ap/PxN3963sJXpr0U7KKuwUl1Xw0oJ0ZvywDcMwXMrvPFjE9pwiT1RVREREk6ql9oy4oAkvLdhGdsFRnvxiE4vSD7I3rxSAdo0jGNghHoD9+aUMnbEEP5OJnyddjjUkwJPVFhGRBkg9RFJrLGZ/7ri0FQAfrchkb14pFnPlf3IvztuKw2Ec+3M6JTY7RWUVpG076LH6iohIw6VAJLVq1IVJJFqD8PczcdelrUibeBnhQWa2ZBfy9fr9bNpXwGer9zjL/7j5gAdrKyIiDZWGzKRWhQSa+er+S6iwO4iLCALgjktaMW1+OtMXpNMkMhjDgNZxYWzPKWJh+kEq7A7M/srqIiJSd/RbR2pddGigMwwB/LlPC6JCAth5sJjF2w4R4G/irVu6Yw0OIK+knDXa4VpEROqYApHUufCgAOcKNIAxF7WgZWwo/dpWHgXy45YcT1VNREQaKAUi8YhberWgRUwICRFBjLu8NQCXt4sD4MfNCkQiIlK3NIdIPCI40J/vUi/FMCr/DNC3TSP8TLD1QCFZR0pOeVaaiIiIu6mHSDwmKMDfGYYAIkMC6d48GoCftuZwtNzOV7/t478rs0g/UOhcpi8iIuJu6iESr3J5+zhW7DrCG2k7mTY/nbyScue9cIuZnq2iuTKlMQM6xGMNdt3A0TAM3l+eSVy4hcEdE+q66iIi4sNMxvFnKMhJFRQUYLVayc/PJyLi1Ce9y7nZdqCQgdMXOd8nWoNIig7htz35lJbbndcD/f0Y1jmRp6/uSKilMtdP+XYzb6TtJNDsx29PDCIowP+E54uISMNS3d/fXj1kNmXKFHr06EF4eDhxcXFcc801bN261aWMYRg8+eSTJCYmEhwcTL9+/di4caNLmbKyMsaNG0dsbCyhoaEMHz6cPXv2IN6ndVwYd13aiiGdEvh/t3Vn8aTL+fiuXqx/chBfjbuY1AHJJMeFYbM7+Gz1Hq57bSl7ckt4I20Hb6TtBMBW4WDT/gIPt0RERHyJV/cQXXHFFdx444306NGDiooKHnvsMdavX8+mTZsIDQ0F4Pnnn+fZZ59l1qxZtGnThmeeeYZFixaxdetWwsPDAbjnnnv48ssvmTVrFjExMYwfP54jR46watUq/P2r14ugHiLvsnznYcZ+uIZDRWWEB5kpPFoBgDU4gPzScp4Y1oE/92np4VqKiIinVff3t1cHouMdPHiQuLg40tLSuPTSSzEMg8TERFJTU5k0aRJQ2RsUHx/P888/z1133UV+fj6NGjXivffe44YbbgBg3759JCUl8c033zB48OBqfbcCkffZl1fKHf/5lY37KnuD7rq0FaEWM9Pmp3NNl0ReurGrh2soIiKeVi+GzI6Xn58PQHR05UqkjIwMsrOzGTRokLOMxWKhb9++LF26FIBVq1ZRXl7uUiYxMZGUlBRnmZMpKyujoKDA5SXeJTEymE/u7sWdl7Zi4uC2PHJlOzonRQKwbk++ZysnIiI+xWcCkWEYPPTQQ1x88cWkpKQAkJ2dDUB8fLxL2fj4eOe97OxsAgMDiYqKOmWZk5kyZQpWq9X5SkpKcmdzxE1CAs3835D2jL2sNSaTic5NrQBkHComr8Tm4dqJiIiv8JlAdN999/Hbb7/x0UcfnXDPZDK5vDcM44RrxztTmUcffZT8/HznKysrq2YVlzoVGRJI85jKDR1/+0MvkcNh4EOjwyIiUsd8IhCNGzeOL774gp9++ommTZs6ryckVO41c3xPT05OjrPXKCEhAZvNRm5u7inLnIzFYiEiIsLlJb6hc9NIANYdOyQ2t9jGJVN/4oY3lmlzRxEROSmvDkSGYXDfffcxZ84cfvzxR1q2dF011LJlSxISEpg/f77zms1mIy0tjd69ewPQrVs3AgICXMrs37+fDRs2OMtI/fL7PKI8AD5Yvpu9eaWs2HWEn3cc8lzFRETEa3n1TtVjx47lww8/5H//+x/h4eHOniCr1UpwcDAmk4nU1FQmT55McnIyycnJTJ48mZCQEEaPHu0se/vttzN+/HhiYmKIjo5mwoQJdOrUiQEDBniyeVJLuiRVziNam5VPWYWdd3/Z7bz37tJdXJLcyFNVExERL+XVgei1114DoF+/fi7X33nnHW677TYAHn74YUpLS7n33nvJzc2lZ8+ezJs3z7kHEcD06dMxm82MHDmS0tJS+vfvz6xZs6q9B5H4lo6JVvz9TBwqKuP1hTs5WFhGVEgAuSXl/LAlh8zDJTSL0cGxIiLyO5/ah8iTtA+Rb7nqlcVs3FdAgL+JcrvBI1e24+fth1i87RB3XtqK/xvS3tNVFBGROlAv9yESqa6qeUTldoOQQH9G9WjGbb1bAPDxyixKbfZTf1hERBocBSKpl7ocW2kGMLJ7EtaQAPq1jaNZdAj5peV8vnbvCZ8pOFrOF+v2cf9Ha+j69Dzu+M+v2LUqTUSkQVAgknqpS7NIAPxM8JdjZ5r5+5kYc1FzAF5P28HBwjJn+eU7D9PnuR+5/6M1fLFuH7kl5czfdICXf9jm8tyDhWXsyyvVnkYiIvWM5hBVk+YQ+Z5/L95JdGgg117w+95V+aXlDJyWRk5hGS1jQ3n/rz3ZkVPEne/9ytFyBy1iQrgipTHhQWb++f1WTCZ4//aeXNQqhhk/buOVH7bhMCAyJIAOjSO4vltTl+eLiIh3qZeHu3qSAlH9setQMTf9ezl780qJC7eQV1KOze7gsraNeO3mbgQFVK4+fOSz35i9MovYMAut40JZtvMIUNnT9MehtFEXJvHk8I5YzFq1KCLibRSI3EyBqH7Zl1fKzW8vZ+fBYgCuTEng5Ru7Emj+fRS51Gbnmn/9zNYDhQCEBvrz7IhOXJGSwPacIr7fmM3Mn7ZjGNC1WSSv39yN+Iggj7RHREROToHIzRSI6p9DRWX8/X8bSIoKYeLgtpj9T5xStz2niNFvLaNxZDDTR3amVaMwl/s/bc3hgY/WUHC0gjbxYXw17hKXUFViqyA4wP+MZ+uJiEjtUCByMwWihsvuMPD3O3Wg2XWomOteW8rhYhsTB7dl7GWtAViy7RB3v7+KC1tG89Yt3U/7DBERqR3ah0jETc4UZFrEhvLYVZUbPb7ywzYyD5eQebiE+z5aTVFZBT9uyeGNRTvO+D1ZR0q46pXFvJF25rIiIuJeCkQibjCiaxN6tYqhrMLBY5+v5873fiWvpJy4cAsA0+als35P/mmf8cK8rWzcV8Dz321hXVZeHdRaRESqKBCJuIHJZOKZESkE+vuxeNshtmQXEhtm4X/39eHKlAQqHAYPzF5Dia3ipJ/fnlPIF+v2AeAw4JE56ym3O+qyCSIiDZoCkYibnNcojLv7nQdAgL+JN8ZcQGNrMJNHdCI+wsLOQ8U89cWmk27q+PIPlavVep8XQ2RIAJv3F/D2koy6boKISIOlQCTiRmMvO4/7LmvNG2O60a15NABRoYFMG9kFkwk+/jWL95btdvlM+oFCvvqtsnfosava89ixg2enz09n16Hium2AiEgDpUAk4kYWsz8TBrfl8nbxLtf7tI5l0hXtAHjqy00s3X7Iee/lH7ZhGDC4YzwdE61c360pvc+rnI80aPoibp+1kv/+msX2nEJsFZXDaIZhkFdiY3tOERUaWhMROWdadl9NWnYv58owDB767zrmrtmLNTiAW3o1Z01mHj/vOIRhwLcPXEL7xpX/bWUdKeGO//zKluxCl2f4maCxNZiC0nIKyyrnI7VLCOeVUV1pEx/uLJdTcJSI4ADnrtsiIg2V9iFyMwUicYej5XZueHPZCavIbr6oGc9c08nlmmEYbMsp4rsN2fywJYftBwopttldygT4myi3G1jMfkwc3BaHYTB3zT427y/AzwTNY0JpGx/On/u0oGermNpunoiI11EgcjMFInGXnIKj/N/cDYRa/OneIpruzaNolxB+xt2sDcPgYGEZmUdKsAYH0DQqhKKyCiZ+uo6FWw+e9rN+Jhg/qC339D0PP20QKSINiAKRmykQibcyDIP//LKbafPTaRsfztVdE7mqU2NsFQ7SDxQxZ/Ue5qzZC0D/dnFMG9kFa0iA8/NHy+38tCWHC1tGExNm8VQzRERqhQKRmykQia8yDIP//prF4//biK3CQdOoYF696QLObxpJ5uES7n5/FZv2FxAZEsD/XdmeP3VvesbeqqPldkpsdsoq7Pj7mYgLP/Whtv/5ZRfv/bKbp4Z3pHfrWHc3T0TktBSI3EyBSHzdhr353PvBajKPlBDo78ef+7Rg9sos8kvLMZmg6l+CC1tG88L1nWkWE+Ly+W0HCpm36QDzNh04YQ7UFR0TeGZECrF/6GEyDINp89OZ8eN2AJpEBrPgob4EB558oveZzowTEakJBSI3UyCS+iC/tJwJn6xj/qYDzmtdm0Xyyo1d+W5DNtPmp1NabicmNJC3b+tBl6RISm12nvhiA//9dc8JzwvwN1HhMDAMiAkN5OmrU+jUxIrNbmfW0l28vywTgDCLmaKyClIHJJM6oI3z8+V2B/M2HuDdpbtYm5XHE8M7cFPP5rX/FyEiDYYCkZspEEl9YRgGby7ayasLd3BNl0T+76r2WMyVvTZ7ciuH0DbsLSA4wJ//G9KO95btJv1AEX4m6NumEQM7JHB5uzjiwi34+ZnYuC+f8f9dd8IWAQAmE/zj6hQiQwK478M1WMx+/DC+L00ig/lk1R6mz09nf/5Rl89Mve58RvZIOuFZJbYK0g8U0bmp9YxDeiIiVRSI3EyBSOobwzBOGiyKyioY+8Fq0tJ/X7nWKNzCyzd2ofd5J58DVFZh55UftvHeL7uxOwwCzH5EhQQyYVBbrjq/MYZhcOOby1iecYS+bRphdxgsObY5ZWxYIKMvbEZuSTnvLduNyQQv3dCFq7s0cT5/w958xn64mt2HS7j5omb84+oUhSIRqRYFIjdTIJKGpNzu4G9zN/Dxr1lc3DqW6Td0oVH4ua1A27SvgKEzFuM49i+OxezHgwPb8Oc+LbCY/TEMg799voEPlmfiZ4L+7eMZ2CGeUpudZ7/Z7NylG+D+/sk8NLDNKb5JROR3CkRupkAkDdGBgqPEhVvc1hvzj6828faSDC5sGc3z151Py9hQl/sOh8H/zV3P7JVZJ3x2QPs4erSIZsq3WwB4+Iq2JFqDWbnrCPvySrmgWRSXtGlEpybWk07OrrA7WLz9EJ+v2cvW7EKaRYfQOi7M+TqvURgB/n5s2JfPr7uOkF9aTv/28XRNilRvlIgPUyByMwUikXNnGAaZR0pIigo57QaRm/YVMH/TAeZvzmbbgSIeGtiGOy9thclk4pUftjFtfvopP2sNDqBP6xguSW5Em/gwNu4rYE1mHou3HeRQke209ava+fuPmseEcE2XJtx0UbPTbi9Q5cctB1iTmceoC5uRGBkMQKnNzocrMtlxsIgOjSPokhRJu4RwzP46TlKktikQuZkCkYhnHD/XyTAMnvt2C+/8vIv2iRF0bx5Fk8hgVmQc4ecdhyg8WnHKZ8WEBjKscyK9z4thX14p2w8Wse1AETsOFjnDUlRIAN2aRxMc6M8Pmw9Qcuy4lECzH9dd0JSbelYGnYgg8wmBZvnOw9z07+VUOAwC/f0YdWESrRqF8a+ftpNTWOZSNiLIzFXnN+aaLk3o0SL6rHYQdzgM7TguUk0KRG6mQCTi/SrsDn7bm8/i9EMs2X6QXYdLaN84gguaRdKteRQXtYoh4BS9MnklNgpKK0iKDnYGsBJbBfM3VW4LsDoz74TPJEUH88TQjgzoEM++vFKGz1zCoSIbjcItHDwuADWJDObKlAS2ZBeyLivPeTgvwHmNQnljTHdax4Wdtn1VK/pKy+188NeeNI0KOW35kzEMg6PlDkpsFYRazDoAWOo9BSI3UyASadhW7jrCG2k7WLbzCEVlrr1QI7s3ZUt2Ib/tyad94wjm3NObNZm5vLRgG3vzSrmrbytu6JHk3N7A7jBYnnGYz9fs5dv12RSWVRAZEsDbt/agW/OoE77bMAzeXbqLyd9swWavnFzeLiGcz+7pTajFTE7hUSZ+8hvr9+YTGRJATGggUSGBRIdWvkpsdrblFLLtQBEHi8qcm3AGBfhxSXIjBh6bwB4VGnhOf0f5peVYzH4KWeJVFIjcTIFIRKqU2x3klZTz1uKdvLV4pzNgRIYE8OV9F5MUXf2emyPFNv4yayVrs/IICvBjbL/WZB4pYVVmLjkFZQT4m/AzmThcXDmkd3m7OH7bk8+hojIGdYjn/v7J3PmfX9l33H5OZys8yMyTwzpy7QVNzjiJvMLuoKisAn8/EyaTiSXbDvLpqj38tPUgCRFBvDGmGylNrGf1/Rv35dMo3FKteVoiZ0OByM0UiETkZJbvPMz4T9ZxsLCMt2/twcXJZ39eW4mtgvs+XMOPW3JOWSbQ34//G9KOW3u3YHVmHqPeXIbN7sDPBA4DWjUK5blrz8dhGBwptnGk2EZusY3DxTYsZj9ax4WRHB9OojWIEIuZ4AB/tmYXsmDzAb76bR/pB4oAGNQhnnGXJ1PucFBqs9MiNpQmxyaHAyzZdogH/7v2hCHBPwoK8OP5685ncMcEft5+iJ+25lB4tILgAH+CAvy5vF0cl7Zp5Cz/wfLdPDZ3A+EWM6+M6spl7eKc96p+Rblrpd8X6/bxz++3MKJLEx4a1Nble9Zm5dEmPpxQi/mMzykuq+DLdfu4tE0j5+R58U4KRG6mQCQip1Jud1BcVkFkSM2HnCrsDp7/bgub9hfQJalyzlOLmFAqHAa2CgcJ1iCXs+LmrN7DQ/9dB8AlybHMHH0B1uCAGn/3G4t28tKC9BNW2fn7mbi2axPGXZ7Ml7/t48V5W517SVWJj7Bw7QVNuTIlgenz0/lpa+WmnkEBfhwtd3Ay9/dPJrV/Mt9uyOa+j1Y7e9lMJpgwqC3dmkfx2ao9fLshG1uFwzn816d1DH+9pBXxEWfXk1Rqs/P0Vxv5aMXvWzq8etMFDOnUGIDnvt3C62k7aBUbygd39KSx9dQhp9Rm57Z3VrA84wiNrUF8cnevGs3ncqc5q/fw31+zeGhgWy5sGe3RungbBSI3UyASEW/zv7V7OVxk45Zezd2yhH/TvgL+/r8N7DxUTEigP4H+fuw8VHxCuRu6J/HE8A74+5mosBuEBPo7e3DsDoNp87fyr592ANDYGsTADvE0iw7haLmdHQeLmbtmLwC9z4th5a4jlNsNRl3YDD8TfLA884z1DPT340/dm9KqURj780rZX3CUgtJyCo5WUFJWQWJkMJ2TIjm/iZXcEhub9xeycGsOOw8VYzJB16RIVmfmEW4x8/X9l7AwPYe//2+j8/nNokP48I6TT1ovq7Bzx39WsegPO7m3jA3lk7t7uQTW4+UW21idmUuTqGCS48JPulfW9xuzeT1tB2P7tWZAh/gz/j1AZRh/5qtNvPvLbgDCLWY+uacX7RIiMAyDD5Zn8tnqPWf1zPpGgcjNFIhEpCFanZnL9PnpLN52iEB/P56+uiM3XtjsjJ9bl5WHv5+JjokRJwx3ffJrFo/N3eCcIH5Vp8a8Mqor/n4mPlyeyZNfbMRi9uOq8xszomsTmkQFc6TYRtaRUmYtzWDlrtwatSU2zMJLN3ShZ6tobnjjF1Zn5tE8JoTMIyUYBtx+cUvmbzpA5pESmkQGc0OPJAqPllNUVkFIoJno0EBW7jrCwq0HCQ7w559/Op8p32xhb14pHRpHcH23puw6XMy+vFIiQwJJjAwm3GJm0baDLN1xGPuxrrXgAH86NbVy80XNGXZ+Y0wmE5+t2sPET9fhMCp71j69u/cZ52HtyyvlwY/XsjzjCFAZ5DKPlJAQEcTsOy/iXz9t55NVvx/K/OCANoy7vPUJWzYcLbez+3AJbeLD6uUmpApEbqZAJCIN2Ya9+YRZzLQ4bnfxmlq1+wgTP/mN9o0jmHZDZ+cKPKjcAiHo2Hyjk1m+8zDvL8/E4TBobA2icWQw0aEBhFkCCA7wZ+ehItZm5bFpXwGRIQG0bxxB+4QIl5V0e3JLuOqVJeSXlgMw6sJmTB6RwoGCMka/teykPWNVAs1+vHNbD/q0jiXjUDF/ev0XDhWdek5VlZaxoeQUHKX42N5WAF2SIrm4dSwzf9oOVIa2Q0VlNLYG8cV9FxMTGsiCzQf4aWsOrWLDuLBlNDFhgby5aCezV2RhszsIs5iZNrIzF7aM5vrXf2F7ThFmPxMVDgM/E/RpHcvibZVnBw7qEM8Twzs654Wt2p3LhE/WkXGomD/3acHfh3ZwhqJfdx3hw+WZHCmxOf+eBnaIZ0TXJsSHB7Fk+yE+WpHJ9pwi+rZpxLUXNKVDYgSHispYvyefvFIb/drEVWv1YomtgjWZeVzYMvqUW2PUlAKRmykQiYjULws2HWDcR2u4vF0cL9/YxTnseLCwjDfSdlBsqyA8KIDQQDMltgqOFNs4WuHgpp7NuKhVjPM5m/cX8OzXmwkPqgyMiZHBFJSWsye3lMNFZXRtFsWVKQm0iA3F7jDYebCIb9Zn88aiHc6NPwFu692CBwe0YcRrP7PzYDEdGkdQbnewLafolG24sGU0k0ek0DouHKgMeiNeXcrBwjKswQHMHN2VS5Ib8d9fs/jbsV45P1PlasWmUSH855ddLnPCxl3emvGD2vLRikwe/3wDFcdPGKNynldMqOWkITAyJIC8knLn+wB/E5e3i+Oq8xNpGhVMbKiFQLMfe3JLyMotYUt2ISsyjrB+Tz4VDoPPx/ahS1JktX+G1aFA5GYKRCIi9c/RcrvH9k3KKTjKtPnp/G/tPu68tBWpA5IxmUzsPFjENf/6mYJju66HW8xc07UJ+/NLWZFxhIKjFfRoEcWDA9rQ67yYE4a5dhwsYs7qPdzQvRnNYn6fB7UuK4/nv9vC0h2HXcpfe0ET2saHO88JvKhVNMt2Vg7DXZmSwGVt44gIDiC3xMbcNXtZcWyILjzIzHUXNKVrs0i+25DND5tzsNkdmExwXqMwzH4mtmQXVvvvo7E1iGdHpHB5O/fOdVIgcjMFIhERqQ3HH08DsGznYabNT6dvm0aM6dWciKDKFYR2h0FeiY3o0MAaz/fZcbCID5dnsiYzl7v6nsfgjgkAvLpwO1O/2+os9+CANtzfv/UJ35N1pISMQ8X0aFF5xE2VvBIbGYeKaR0XRvix+m7JLuCzVXtYsSuXw0VlHCoqw1bhoLE1mKToYFrGhtK9eTQXtoymaVRwrcxhUiByMwUiERGp72b+uI1PVu1hwqC2DOuc6PbnG4aBYVCnZ/EpELmZApGIiIjvqe7vb/dO5RYRERHxQQpEIiIi0uApEImIiEiDp0AkIiIiDZ4CkYiIiDR4CkQiIiLS4CkQiYiISIPXoALRq6++SsuWLQkKCqJbt24sXrzY01USERERL9BgAtHHH39Mamoqjz32GGvWrOGSSy7hyiuvJDMz09NVExEREQ9rMDtV9+zZkwsuuIDXXnvNea19+/Zcc801TJky5YTyZWVllJX9fpJvQUEBSUlJ2qlaRETEh2in6j+w2WysWrWKQYMGuVwfNGgQS5cuPelnpkyZgtVqdb6SkpLqoqoiIiLiAQ0iEB06dAi73U58fLzL9fj4eLKzs0/6mUcffZT8/HznKysrqy6qKiIiIh5g9nQF6pLJ5Hq6rmEYJ1yrYrFYsFgsdVEtERER8bAG0UMUGxuLv7//Cb1BOTk5J/QaiYiISMPTIHqIAgMD6datG/Pnz2fEiBHO6/Pnz+fqq6+u1jOq5p4XFBTUSh1FRETE/ap+b59pDVmDCEQADz30EGPGjKF79+706tWLN998k8zMTO6+++5qfb6wsBBAk6tFRER8UGFhIVar9ZT3G0wguuGGGzh8+DBPP/00+/fvJyUlhW+++YbmzZtX6/OJiYlkZWURHh5+ynlHvqpqS4GsrKwGsaWA2lt/NaS2gtpb3zWk9tZmWw3DoLCwkMTExNOWazD7EMmpVXePhvpC7a2/GlJbQe2t7xpSe72hrQ1iUrWIiIjI6SgQiYiISIOnQCRYLBaeeOKJBrPvktpbfzWktoLaW981pPZ6Q1s1h0hEREQaPPUQiYiISIOnQCQiIiINngKRiIiINHgKRCIiItLgKRA1EFOmTKFHjx6Eh4cTFxfHNddcw9atW13KGIbBk08+SWJiIsHBwfTr14+NGzd6qMbuNWXKFEwmE6mpqc5r9a29e/fu5eabbyYmJoaQkBC6dOnCqlWrnPfrS3srKir429/+RsuWLQkODqZVq1Y8/fTTOBwOZxlfbuuiRYsYNmwYiYmJmEwmPv/8c5f71WlbWVkZ48aNIzY2ltDQUIYPH86ePXvqsBXVd7r2lpeXM2nSJDp16kRoaCiJiYnccsst7Nu3z+UZ9aW9x7vrrrswmUy89NJLLtd9pb3VaevmzZsZPnw4VquV8PBwLrroIjIzM53367KtCkQNRFpaGmPHjmXZsmXMnz+fiooKBg0aRHFxsbPM1KlTmTZtGjNnzmTlypUkJCQwcOBA5zluvmrlypW8+eabnH/++S7X61N7c3Nz6dOnDwEBAXz77bds2rSJF198kcjISGeZ+tLe559/ntdff52ZM2eyefNmpk6dyj//+U9mzJjhLOPLbS0uLqZz587MnDnzpPer07bU1FTmzp3L7NmzWbJkCUVFRQwdOhS73V5Xzai207W3pKSE1atX8/jjj7N69WrmzJlDeno6w4cPdylXX9r7R59//jnLly8/6XETvtLeM7V1x44dXHzxxbRr146FCxeybt06Hn/8cYKCgpxl6rSthjRIOTk5BmCkpaUZhmEYDofDSEhIMJ577jlnmaNHjxpWq9V4/fXXPVXNc1ZYWGgkJycb8+fPN/r27Ws88MADhmHUv/ZOmjTJuPjii095vz6196qrrjL+8pe/uFy79tprjZtvvtkwjPrVVsCYO3eu83112paXl2cEBAQYs2fPdpbZu3ev4efnZ3z33Xd1VveaOL69J7NixQoDMHbv3m0YRv1s7549e4wmTZoYGzZsMJo3b25Mnz7dec9X23uytt5www3O/92eTF23VT1EDVR+fj4A0dHRAGRkZJCdnc2gQYOcZSwWC3379mXp0qUeqaM7jB07lquuuooBAwa4XK9v7f3iiy/o3r07f/rTn4iLi6Nr16689dZbzvv1qb0XX3wxP/zwA+np6QCsW7eOJUuWMGTIEKB+tfV41WnbqlWrKC8vdymTmJhISkqKz7cfKv/tMplMzt7P+tZeh8PBmDFjmDhxIh07djzhfn1pr8Ph4Ouvv6ZNmzYMHjyYuLg4evbs6TKsVtdtVSBqgAzD4KGHHuLiiy8mJSUFgOzsbADi4+NdysbHxzvv+ZrZs2ezevVqpkyZcsK9+tbenTt38tprr5GcnMz333/P3Xffzf33389//vMfoH61d9KkSYwaNYp27doREBBA165dSU1NZdSoUUD9auvxqtO27OxsAgMDiYqKOmUZX3X06FEeeeQRRo8e7TwAtL619/nnn8dsNnP//fef9H59aW9OTg5FRUU899xzXHHFFcybN48RI0Zw7bXXkpaWBtR9W81uf6J4vfvuu4/ffvuNJUuWnHDPZDK5vDcM44RrviArK4sHHniAefPmuYxHH6++tNfhcNC9e3cmT54MQNeuXdm4cSOvvfYat9xyi7NcfWjvxx9/zPvvv8+HH35Ix44dWbt2LampqSQmJnLrrbc6y9WHtp5KTdrm6+0vLy/nxhtvxOFw8Oqrr56xvC+2d9WqVbz88susXr36rOvua+2tWgRx9dVX8+CDDwLQpUsXli5dyuuvv07fvn1P+dnaaqt6iBqYcePG8cUXX/DTTz/RtGlT5/WEhASAE1J3Tk7OCf9v1BesWrWKnJwcunXrhtlsxmw2k5aWxiuvvILZbHa2qb60t3HjxnTo0MHlWvv27Z2rNerTz3fixIk88sgj3HjjjXTq1IkxY8bw4IMPOnsC61Nbj1edtiUkJGCz2cjNzT1lGV9TXl7OyJEjycjIYP78+c7eIahf7V28eDE5OTk0a9bM+e/W7t27GT9+PC1atADqT3tjY2Mxm81n/HerLtuqQNRAGIbBfffdx5w5c/jxxx9p2bKly/2WLVuSkJDA/PnznddsNhtpaWn07t27rqt7zvr378/69etZu3at89W9e3duuukm1q5dS6tWrepVe/v06XPCNgrp6ek0b94cqF8/35KSEvz8XP/p8vf3d/4/zvrU1uNVp23dunUjICDApcz+/fvZsGGDT7a/Kgxt27aNBQsWEBMT43K/PrV3zJgx/Pbbby7/biUmJjJx4kS+//57oP60NzAwkB49epz23606b6vbp2mLV7rnnnsMq9VqLFy40Ni/f7/zVVJS4izz3HPPGVar1ZgzZ46xfv16Y9SoUUbjxo2NgoICD9bcff64ysww6ld7V6xYYZjNZuPZZ581tm3bZnzwwQdGSEiI8f777zvL1Jf23nrrrUaTJk2Mr776ysjIyDDmzJljxMbGGg8//LCzjC+3tbCw0FizZo2xZs0aAzCmTZtmrFmzxrmqqjptu/vuu42mTZsaCxYsMFavXm1cfvnlRufOnY2KigpPNeuUTtfe8vJyY/jw4UbTpk2NtWvXuvzbVVZW5nxGfWnvyRy/yswwfKe9Z2rrnDlzjICAAOPNN980tm3bZsyYMcPw9/c3Fi9e7HxGXbZVgaiBAE76euedd5xlHA6H8cQTTxgJCQmGxWIxLr30UmP9+vWeq7SbHR+I6lt7v/zySyMlJcWwWCxGu3btjDfffNPlfn1pb0FBgfHAAw8YzZo1M4KCgoxWrVoZjz32mMsvSF9u608//XTS/63eeuuthmFUr22lpaXGfffdZ0RHRxvBwcHG0KFDjczMTA+05sxO196MjIxT/tv1008/OZ9RX9p7MicLRL7S3uq09e233zZat25tBAUFGZ07dzY+//xzl2fUZVtNhmEY7u93EhEREfEdmkMkIiIiDZ4CkYiIiDR4CkQiIiLS4CkQiYiISIOnQCQiIiINngKRiIiINHgKRCIiItLgKRCJiIhIg6dAJCJeq0WLFrz00kvVLr9w4UJMJhN5eXm1VicRqZ+0U7WIuE2/fv3o0qXLWYWY0zl48CChoaGEhIRUq7zNZuPIkSPEx8djMpncUoeztXDhQi677DJyc3OJjIz0SB1E5OyZPV0BEWlYDMPAbrdjNp/5n59GjRqd1bMDAwNJSEioadVEpAHTkJmIuMVtt91GWloaL7/8MiaTCZPJxK5du5zDWN9//z3du3fHYrGwePFiduzYwdVXX018fDxhYWH06NGDBQsWuDzz+CEzk8nEv//9b0aMGEFISAjJycl88cUXzvvHD5nNmjWLyMhIvv/+e9q3b09YWBhXXHEF+/fvd36moqKC+++/n8jISGJiYpg0aRK33nor11xzzSnbunv3boYNG0ZUVBShoaF07NiRb775hl27dnHZZZcBEBUVhclk4rbbbgMqg+DUqVNp1aoVwcHBdO7cmU8//fSEun/99dd07tyZoKAgevbsyfr168/4vSJy7hSIRMQtXn75ZXr16sUdd9zB/v372b9/P0lJSc77Dz/8MFOmTGHz5s2cf/75FBUVMWTIEBYsWMCaNWsYPHgww4YNIzMz87Tf89RTTzFy5Eh+++03hgwZwk033cSRI0dOWb6kpIQXXniB9957j0WLFpGZmcmECROc959//nk++OAD3nnnHX7++WcKCgr4/PPPT1uHsWPHUlZWxqJFi1i/fj3PP/88YWFhJCUl8dlnnwGwdetW9u/fz8svvwzA3/72N9555x1ee+01Nm7cyIMPPsjNN99MWlqay7MnTpzICy+8wMqVK4mLi2P48OGUl5ef9ntFxA0MERE36du3r/HAAw+4XPvpp58MwPj888/P+PkOHToYM2bMcL5v3ry5MX36dOd7wPjb3/7mfF9UVGSYTCbj22+/dfmu3NxcwzAM45133jEAY/v27c7P/Otf/zLi4+Od7+Pj441//vOfzvcVFRVGs2bNjKuvvvqU9ezUqZPx5JNPnvTe8XWoqmdQUJCxdOlSl7K33367MWrUKJfPzZ4923n/8OHDRnBwsPHxxx+f8XtF5NxoDpGI1Inu3bu7vC8uLuapp57iq6++Yt++fVRUVFBaWnrGHqLzzz/f+efQ0FDCw8PJyck5ZfmQkBDOO+885/vGjRs7y+fn53PgwAEuvPBC531/f3+6deuGw+E45TPvv/9+7rnnHubNm8eAAQO47rrrXOp1vE2bNnH06FEGDhzoct1ms9G1a1eXa7169XL+OTo6mrZt27J58+Yafa+IVJ+GzESkToSGhrq8nzhxIp999hnPPvssixcvZu3atXTq1AmbzXba5wQEBLi8N5lMpw0vJytvHLe49vgVacffP95f//pXdu7cyZgxY1i/fj3du3dnxowZpyxfVb+vv/6atWvXOl+bNm1ymUd0KlX1O9vvFZHqUyASEbcJDAzEbrdXq+zixYu57bbbGDFiBJ06dSIhIYFdu3bVbgWPY7VaiY+PZ8WKFc5rdrudNWvWnPGzSUlJ3H333cyZM4fx48fz1ltvAZV/B1XPqdKhQwcsFguZmZm0bt3a5fXHeVYAy5Ytc/45NzeX9PR02rVrd8bvFZFzoyEzEXGbFi1asHz5cnbt2kVYWBjR0dGnLNu6dWvmzJnDsGHDMJlMPP7446ft6akt48aNY8qUKbRu3Zp27doxY8YMcnNzT7uPUWpqKldeeSVt2rQhNzeXH3/8kfbt2wPQvHlzTCYTX331FUOGDCE4OJjw8HAmTJjAgw8+iMPh4OKLL6agoIClS5cSFhbGrbfe6nz2008/TUxMDPHx8Tz22GPExsY6V7yd7ntF5Nyoh0hE3GbChAn4+/vToUMHGjVqdNr5QNOnTycqKorevXszbNgwBg8ezAUXXFCHta00adIkRo0axS233EKvXr0ICwtj8ODBBAUFnfIzdrudsWPH0r59e6644gratm3Lq6++CkCTJk146qmneOSRR4iPj+e+++4D4B//+Ad///vfmTJlCu3bt2fw4MF8+eWXtGzZ0uXZzz33HA888ADdunVj//79fPHFFy69Tqf6XhE5N9qpWkTkDxwOB+3bt2fkyJH84x//qLPv1Q7XIp6lITMRadB2797NvHnz6Nu3L2VlZcycOZOMjAxGjx7t6aqJSB3SkJmINGh+fn7MmjWLHj160KdPH9avX8+CBQs0N0ekgdGQmYiIiDR46iESERGRBk+BSERERBo8BSIRERFp8BSIREREpMFTIBIREZEGT4FIREREGjwFIhEREWnwFIhERESkwfv/r6SQsWQQ/aMAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from torch.optim import SGD, Adam\n",
    "from copy import deepcopy\n",
    "\n",
    "# 将数据适配为PyTorch数据集\n",
    "class SeqLabelDataset(Dataset):\n",
    "    def __init__(self, tokens, labels):\n",
    "        self.tokens = deepcopy(tokens)\n",
    "        self.labels = deepcopy(labels)\n",
    "        \n",
    "    def __getitem__(self, idx):\n",
    "        return self.tokens[idx], self.labels[idx]\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.labels)\n",
    "    \n",
    "    # 将每个批次转化为PyTorch张量\n",
    "    @classmethod\n",
    "    def collate_batch(cls, inputs):\n",
    "        input_ids, labels, masks = [], [], []\n",
    "        max_len = -1\n",
    "        for ids, tags in inputs:\n",
    "            input_ids.append(ids)\n",
    "            labels.append(tags)\n",
    "            masks.append([1] * len(tags))\n",
    "            max_len = max(max_len, len(ids))\n",
    "        for ids, tags, msks in zip(input_ids, labels, masks):\n",
    "            pad_len = max_len - len(ids)\n",
    "            ids.extend([0] * pad_len)\n",
    "            tags.extend([0] * pad_len)\n",
    "            msks.extend([0] * pad_len)\n",
    "        input_ids = torch.tensor(np.array(input_ids),\\\n",
    "            dtype=torch.long)\n",
    "        labels = torch.tensor(np.array(labels), dtype=torch.long)\n",
    "        masks = torch.tensor(np.array(masks), dtype=torch.uint8)\n",
    "        return {'input_ids': input_ids, 'masks': masks,\\\n",
    "                'labels': labels}\n",
    "    \n",
    "# 准备数据\n",
    "train_dataset = SeqLabelDataset(train_X, train_Y)\n",
    "test_dataset = SeqLabelDataset(test_X, test_Y)\n",
    "\n",
    "from tqdm import tqdm, trange\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "def train(model, batch_size, epochs, learning_rate):\n",
    "    train_dataloader = DataLoader(train_dataset,\n",
    "        batch_size=batch_size, shuffle=True, \n",
    "        collate_fn=SeqLabelDataset.collate_batch)\n",
    "    optimizer = Adam(model.parameters(), lr=learning_rate)\n",
    "    model.zero_grad()\n",
    "    tr_step = []\n",
    "    tr_loss = []\n",
    "    global_step = 0\n",
    "    \n",
    "    with trange(epochs, desc='epoch', ncols=60) as pbar:\n",
    "        for epoch in pbar:\n",
    "            model.train()\n",
    "            for step, batch in enumerate(train_dataloader):\n",
    "                global_step += 1\n",
    "                loss = model(**batch)\n",
    "                loss.backward()\n",
    "                optimizer.step()\n",
    "                model.zero_grad()\n",
    "                pbar.set_description(f'epoch-{epoch}, '+\\\n",
    "                    f'loss={loss.item():.2f}')\n",
    "                \n",
    "                if epoch > 0:\n",
    "                    tr_step.append(global_step)\n",
    "                    tr_loss.append(loss.item())\n",
    "\n",
    "    # 打印损失曲线\n",
    "    plt.plot(tr_step, tr_loss, label='train loss')\n",
    "    plt.xlabel('training steps')\n",
    "    plt.ylabel('loss')\n",
    "    plt.legend()\n",
    "    plt.show()\n",
    "                \n",
    "vocab_size = len(dataset.token2id)\n",
    "hidden_size = 32\n",
    "n_tags = len(dataset.label2id)\n",
    "crf = CRF(vocab_size, hidden_size, n_tags)\n",
    "batch_size = 128\n",
    "epochs = 20\n",
    "learning_rate = 1e-2\n",
    "train(crf, batch_size, epochs, learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "4a1e7e25",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precision = 0.551164616592515, recall = 0.2510729613733906, f1 = 0.34499139978704235\n",
      "precision = 0.4896449704142012, recall = 0.20269442743417024, f1 = 0.2867042009527934\n"
     ]
    }
   ],
   "source": [
    "# 验证效果\n",
    "\n",
    "def evaluate(X, Y, model, batch_size):\n",
    "    dataset = SeqLabelDataset(X, Y)\n",
    "    dataloader = DataLoader(dataset, batch_size=batch_size,\\\n",
    "        collate_fn=SeqLabelDataset.collate_batch)\n",
    "    model.eval()\n",
    "    with torch.no_grad():\n",
    "        P = []\n",
    "        for batch in dataloader:\n",
    "            preds = model.decode(batch['input_ids'], batch['masks'])\n",
    "            P.extend(preds)\n",
    "\n",
    "    p, r, f = compute_metric(Y, P)\n",
    "    print(f'precision = {p}, recall = {r}, f1 = {f}')\n",
    "    \n",
    "evaluate(train_X, train_Y, crf, batch_size)\n",
    "evaluate(test_X, test_Y, crf, batch_size)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cc369a17",
   "metadata": {},
   "source": [
    "\n",
    "\n",
    "这里介绍基于双向长短期记忆-条件随机场（BiLSTM-CRF）结构的代码实现。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "990c5bc1",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch import nn\n",
    "\n",
    "# 定义模型，将长短期记忆的输出输入给条件随机场，\n",
    "# 利用条件随机场计算损失和解码\n",
    "class LSTM_CRF(nn.Module):\n",
    "    def __init__(self, vocab_size, hidden_size, num_layers,\\\n",
    "                 dropout, n_tags):\n",
    "        super().__init__()\n",
    "        self.embedding = nn.Embedding(vocab_size, hidden_size)\n",
    "        self.lstm = nn.LSTM(input_size=hidden_size,\n",
    "                            hidden_size=hidden_size,\n",
    "                            num_layers=num_layers,\n",
    "                            batch_first=False,\n",
    "                            dropout=dropout,\n",
    "                            bidirectional=True)\n",
    "        self.crf = CRFLayer(n_tags, hidden_size * 2)\n",
    "    \n",
    "    def forward(self, input_ids, masks, labels):\n",
    "        \"\"\"\n",
    "        input_ids/masks/labels: batch_size * seq_len\n",
    "        \"\"\"\n",
    "        # 将输入序列转化为词嵌入序列\n",
    "        # batch_size * seq_len * embed_size\n",
    "        embed = self.embedding(input_ids)\n",
    "        embed = torch.transpose(embed, 0, 1)\n",
    "        masks = torch.transpose(masks, 0, 1)\n",
    "        labels = torch.transpose(labels, 0, 1)\n",
    "        # 输入长短期记忆得到隐状态\n",
    "        hidden_states, _ = self.lstm(embed)\n",
    "        # 隐状态和标签、掩码输入条件随机场计算损失\n",
    "        llh = self.crf(hidden_states, labels, masks)\n",
    "        return -llh\n",
    "    \n",
    "    def decode(self, input_ids, masks):\n",
    "        # 输入长短期记忆得到隐状态\n",
    "        embed = self.embedding(input_ids)\n",
    "        embed = torch.transpose(embed, 0, 1)\n",
    "        masks = torch.transpose(masks, 0, 1)\n",
    "        hidden_states, _ = self.lstm(embed)\n",
    "        # 调用条件随机场进行解码\n",
    "        return self.crf.decode(hidden_states, masks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "ab41a9f0",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "epoch-19, loss=41.44: 100%|█| 20/20 [07:28<00:00, 22.44s/it]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAGwCAYAAABPSaTdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABsTklEQVR4nO3dd3zV1f3H8dfNTW72DlkQ9iZsEEEUUARBwdVq3bRqHYDi1o5f0VZR2zoKFuuoW7FVwC2jQACRIXvJDCSBhEDIXjfJ/f7+uLk3uZBFdi7v5+NxH+Xe7/feew5U8uZzlskwDAMRERERN+XR0g0QERERaUoKOyIiIuLWFHZERETErSnsiIiIiFtT2BERERG3prAjIiIibk1hR0RERNyaZ0s3oDWw2WwcP36cwMBATCZTSzdHRERE6sAwDHJzc4mNjcXDo/r6jcIOcPz4ceLi4lq6GSIiIlIPycnJdOjQodrrCjtAYGAgYP/NCgoKauHWiIiISF3k5OQQFxfn/DleHYUdcA5dBQUFKeyIiIi0MbVNQdEEZREREXFrCjsiIiLi1hR2RERExK1pzo6IiLi9srIySkpKWroZco68vLwwm80N/hyFHRERcVuGYZCWlkZWVlZLN0XqKSQkhOjo6Abtg6ewIyIibssRdCIjI/Hz89PGsW2IYRgUFBSQnp4OQExMTL0/S2FHRETcUllZmTPohIeHt3RzpB58fX0BSE9PJzIyst5DWpqgLCIibskxR8fPz6+FWyIN4fjza8icK4UdERFxaxq6atsa489PYUdERETcmsKOiIiIuDWFHRERETfXuXNnXnnllRb/jJai1VgtpNBahq+l4RsliYiI+xk7diyDBg1qtHCxadMm/P39G+Wz2iJVdlrAuz8kEj97CQn7T7Z0U0REpI0yDIPS0tI63duuXbvzelWawk4L2JKURZnN4Kcjp1u6KSIi5xXDMCiwlrbIwzCMOrVx2rRpJCQk8Oqrr2IymTCZTBw5coRVq1ZhMplYsmQJw4YNw9vbmzVr1nDo0CGuvvpqoqKiCAgIYPjw4SxfvtzlM88cgjKZTLz11ltce+21+Pn50aNHD7788stz+r1MSkri6quvJiAggKCgIG644QZOnDjhvL59+3bGjRtHYGAgQUFBDB06lJ9++gmAo0ePMmXKFEJDQ/H396dfv358++235/T950LDWC2gsKQMgMwCawu3RETk/FJYUkbf/1vSIt+955mJ+Flq/7H76quvsn//fuLj43nmmWcAe2XmyJEjADz++OP87W9/o2vXroSEhJCSksLkyZP5y1/+go+PD++99x5Tpkxh3759dOzYsdrvefrpp3nxxRf561//yty5c7nllls4evQoYWFhtbbRMAyuueYa/P39SUhIoLS0lPvvv58bb7yRVatWAXDLLbcwePBg5s+fj9lsZtu2bXh5eQEwffp0rFYrq1evxt/fnz179hAQEFDr99aXwk4LKHKGHR1KJyIiroKDg7FYLPj5+REdHX3W9WeeeYbLL7/c+Tw8PJyBAwc6n//lL39h0aJFfPnll8yYMaPa75k2bRo33XQTAM899xxz585l48aNXHHFFbW2cfny5ezYsYPExETi4uIA+OCDD+jXrx+bNm1i+PDhJCUl8dhjj9G7d28AevTo4Xx/UlIS119/Pf379wega9eutX5nQyjstIACa3nYyVdlR0SkOfl6mdnzzMQW++7GMGzYMJfn+fn5PP3003z99dccP36c0tJSCgsLSUpKqvFzBgwY4Py1v78/gYGBznOoarN3717i4uKcQQegb9++hISEsHfvXoYPH87DDz/MXXfdxQcffMD48eP55S9/Sbdu3QB44IEHuO+++1i6dCnjx4/n+uuvd2lPY9OcnRZQaFVlR0SkJZhMJvwsni3yaKydnM9cVfXYY4/x+eef8+yzz7JmzRq2bdtG//79sVpr/ge1Y0ip8u+NzWarUxsMw6iyP5Vfnz17Nrt37+bKK69kxYoV9O3bl0WLFgFw1113cfjwYW677TZ27tzJsGHDmDt3bp2+uz4UdlqAcxhLlR0REamCxWKhrKysTveuWbOGadOmce2119K/f3+io6Od83uaSt++fUlKSiI5Odn52p49e8jOzqZPnz7O13r27MlDDz3E0qVLue6663jnnXec1+Li4rj33ntZuHAhjzzyCG+++WaTtVdhpwVogrKIiNSkc+fObNiwgSNHjnDq1KkaKy7du3dn4cKFbNu2je3bt3PzzTfXuUJTX+PHj2fAgAHccsstbNmyhY0bN3L77bczZswYhg0bRmFhITNmzGDVqlUcPXqUH374gU2bNjmD0KxZs1iyZAmJiYls2bKFFStWuISkxqaw0wIcc3aKS23OIS0RERGHRx99FLPZTN++fWnXrl2N829efvllQkNDGTVqFFOmTGHixIkMGTKkSdtnMplYvHgxoaGhXHLJJYwfP56uXbvy6aefAmA2m8nIyOD222+nZ8+e3HDDDUyaNImnn34agLKyMqZPn06fPn244oor6NWrF//85z+brr1GXRf+u7GcnByCg4PJzs4mKCioyb+v5x++w1pqT90/PHkp7UN8m/w7RUTON0VFRSQmJtKlSxd8fHxaujlSTzX9Odb157cqO82szGY4gw5o3o6IiEhTU9hpZo7JyQ6atyMiItK0FHaaWYH1zLCj5eciIiJNSWGnmZ1Z2clSZUdEpElpamrb1hh/fgo7zazwjLBzWnN2RESahGPTvIKCghZuiTSE48/vzE0Qz4WOi2hmZy41z9IwlohIkzCbzYSEhDiPQPDz82u0XYyl6RmGQUFBAenp6YSEhGA21/+4DYWdZqbKjohI83EcpFnXM5+k9QkJCanyQNRzobDTzM6s7Gg1lohI0zGZTMTExBAZGUlJiSrpbY2Xl1eDKjoOCjvN7MzKjoaxRESantlsbpQfmtI2aYJyM3NUdgJ97DlTw1giIiJNS2GnmTkqO44jIiovPS+zGSSf1qoBERGRxqSw08wclZ3Y8rCTby2juNT+2t+W7uPiF1eybM+JFmufiIiIu1HYaWaOyk5koDce5SsgHfN21h/OAGBnSlZLNE1ERMQttWjYmT9/PgMGDCAoKIigoCBGjhzJd99957xuGAazZ88mNjYWX19fxo4dy+7du10+o7i4mJkzZxIREYG/vz9Tp04lJSWlubtSZ46w42fxJNTPAthXZBmGwcH0PADSc4tbrH0iIiLupkXDTocOHXj++ef56aef+Omnn7j00ku5+uqrnYHmxRdf5KWXXmLevHls2rSJ6OhoLr/8cnJzc52fMWvWLBYtWsSCBQtYu3YteXl5XHXVVZSVlVX3tS3KMYzla/EgxM++G+TpfCvpucXkFpUCCjsiIiKNqUXDzpQpU5g8eTI9e/akZ8+ePPvsswQEBLB+/XoMw+CVV17h97//Pddddx3x8fG89957FBQU8PHHHwOQnZ3N22+/zd///nfGjx/P4MGD+fDDD9m5cyfLly9vya5VyxF2Kld2sgpKnFUdgPTcohZpm4iIiDtqNXN2ysrKWLBgAfn5+YwcOZLExETS0tKYMGGC8x5vb2/GjBnDunXrANi8eTMlJSUu98TGxhIfH++8pyrFxcXk5OS4PJqLYxjLx8tMqL897JzOt7qEnRM5quyIiIg0lhYPOzt37iQgIABvb2/uvfdeFi1aRN++fUlLSwMgKirK5f6oqCjntbS0NCwWC6GhodXeU5U5c+YQHBzsfMTFxTVyr6rnCDu+XmZCy4exsgpcw05GXjFlNp3SKyIi0hhaPOz06tWLbdu2sX79eu677z7uuOMO9uzZ47x+5qFthmHUepBbbfc89dRTZGdnOx/JyckN68Q5KCqpmLPjqOxkFpRwIL1iHpLNsAceERERabgWDzsWi4Xu3bszbNgw5syZw8CBA3n11Vedh36dWaFJT093Vnuio6OxWq1kZmZWe09VvL29nSvAHI/mUmCtXNkpDzv5Vg6m57vcp6EsERGRxtHiYedMhmFQXFxMly5diI6OZtmyZc5rVquVhIQERo0aBcDQoUPx8vJyuSc1NZVdu3Y572ltKlZjeRJWHnYSM/I5VV7J6RrhD2iSsoiISGNp0YNAf/e73zFp0iTi4uLIzc1lwYIFrFq1iu+//x6TycSsWbN47rnn6NGjBz169OC5557Dz8+Pm2++GYDg4GDuvPNOHnnkEcLDwwkLC+PRRx+lf//+jB8/viW7Vq2iSnN2HEvPdx3LBiA22IcuEf4cPpWv5eciIiKNpEXDzokTJ7jttttITU0lODiYAQMG8P3333P55ZcD8Pjjj1NYWMj9999PZmYmI0aMYOnSpQQGBjo/4+WXX8bT05MbbriBwsJCLrvsMt59991We7pt5QnKpvI5OyVl9snI3SIDiAzyBiBdw1giIiKNokXDzttvv13jdZPJxOzZs5k9e3a19/j4+DB37lzmzp3byK1rGoWVJij7WlwDWY/IQALKT0M/oWEsERGRRtGiYed8VFBpzo6Pp+uUqe6RAdgMe5VHlR0REZHGobDTjMpsBtZSG2Afxgry8cRkgvJ8Q/fIALIL7YeCnlRlR0REpFG0utVY7swxORnsYcfT7EGQj5fztR6RAUQG2ufsaOm5iIhI41DYaUaFlcKOd/kQlmMX5XB/C6H+FucE5VN5xdi0i7KIiEiDKew0o8JKGwp6eNh3eHbsotw9MgCAiABvTCYotRmcLrC2TENFRETciMJOM6pYiVWxCsuxi7Ij7HiZPQgvD0CapCwiItJwCjvNqHJlx8GxY/LgjhWHmbYL9AG0/FxERKQxaDVWM3JUdny8KjLmIxN6cXnfKIZ1DnO+Fhnozd5UOKnKjoiISIMp7DSjinOxKio7vhYzI7qGu9wX5dhFWZUdERGRBtMwVjNyVHb8vGrOmJGOYSxVdkRERBpMYacZOSo7Ppaaz+2KVGVHRESk0SjsNKOKQ0Br/m13bCyok89FREQaTmGnGVW1GqsqkUH2YSwtPRcREWk4hZ1mVLHPTm1zduyVnZO5xRiGdlEWERFpCIWdZlQxjFVzZaddedixltnIKihp8naJiIi4M4WdZlSx9Lzm33ZvTzMh5Wdmad6OiIhIwyjsNKOiOlZ2AKKcy8+1IktERKQhFHaaUYG1bnN2oPLyc1V2REREGkJhpxnVdc4O4DwMNDNfJ5+LiIg0hMJOM3IOY9UyZwcgyNc+ZyenSBOURUREGkJhpwklny5g+Z4TJJ7KB+q+zw5AkE952ClU2BEREWkIhZ0m9Ncl+7jr/Z9YsjsNOLc5O8HOyk5p0zVQRETkPKCw04S6RPgDcPhkHnBuq7GCfO2BSJUdERGRhlHYaUJd29nDjnMY61zCjo/m7IiIiDQGhZ0m1K1dAACHT54Rds5hgnK2KjsiIiINorDThDqXD2Nl5FvJLihxztnxOacJypqzIyIi0hAKO00owNuTqPLNAQ+ezMVaagPArw4TlJ1zdjSMJSIi0iAKO02sa4R9KGvP8Rzna+cyZ6fAWkZJma1pGiciInIeUNhpYl3KJynvSa0IO96etf+2B/pUVH9ytfxcRESk3hR2mljX8nk7u8srOz5eHnh4mGp9n6fZgwBvLT8XERFpKIWdJuZYfv5zWi5Qt/k6DkE+mrcjIiLSUAo7TcwxZ8cxObku83UcnOdjaUWWiIhIvSnsNLEOob54mSuGrXy86v5bro0FRUREGk5hp4l5mj3oGObnfO5rOZfKjubsiIiINJTCTjPoWr6TMoCf17nM2dEuyiIiIg2lsNMMHJOUAXzOqbKjYSwREZGGUthpBo7l5wC+5zRnxzGMpQnKIiIi9aWw0wwqD2PVazWWKjsiIiL1prDTDLpUruzUZxhLc3ZERETqTWGnGYT7W5xDUr71mKCco+MiRERE6k1hpxmYTCbnUJav5Rzm7GjpuYiISIMp7DSTHpH2sBNcPjRVF9pUUEREpOHqPqYiDTLj0u5EBflw/ZAOdX5PsI6LEBERaTCFnWbSKdyfRyf2Oqf3OCo7hSVlWEttWDxViBMRETlX+unZigX4VGRRDWWJiIjUj8JOK2b2MBHorUnKIiIiDdGiYWfOnDkMHz6cwMBAIiMjueaaa9i3b5/LPdOmTcNkMrk8LrzwQpd7iouLmTlzJhEREfj7+zN16lRSUlKasytNpmJjQc3bERERqY8WDTsJCQlMnz6d9evXs2zZMkpLS5kwYQL5+fku911xxRWkpqY6H99++63L9VmzZrFo0SIWLFjA2rVrycvL46qrrqKsrKw5u9MktLGgiIhIw7ToBOXvv//e5fk777xDZGQkmzdv5pJLLnG+7u3tTXR0dJWfkZ2dzdtvv80HH3zA+PHjAfjwww+Ji4tj+fLlTJw4sek60Ayc52Npzo6IiEi9tKo5O9nZ2QCEhYW5vL5q1SoiIyPp2bMnd999N+np6c5rmzdvpqSkhAkTJjhfi42NJT4+nnXr1lX5PcXFxeTk5Lg8WqsgLT8XERFpkFYTdgzD4OGHH2b06NHEx8c7X580aRIfffQRK1as4O9//zubNm3i0ksvpbi4GIC0tDQsFguhoaEunxcVFUVaWlqV3zVnzhyCg4Odj7i4uKbrWANpY0EREZGGaTX77MyYMYMdO3awdu1al9dvvPFG56/j4+MZNmwYnTp14ptvvuG6666r9vMMw8BkMlV57amnnuLhhx92Ps/JyWm1gUdHRoiIiDRMq6jszJw5ky+//JKVK1fSoUPNOwzHxMTQqVMnDhw4AEB0dDRWq5XMzEyX+9LT04mKiqryM7y9vQkKCnJ5tFaq7IiIiDRMi4YdwzCYMWMGCxcuZMWKFXTp0qXW92RkZJCcnExMTAwAQ4cOxcvLi2XLljnvSU1NZdeuXYwaNarJ2t5cNGdHRESkYVp0GGv69Ol8/PHHfPHFFwQGBjrn2AQHB+Pr60teXh6zZ8/m+uuvJyYmhiNHjvC73/2OiIgIrr32Wue9d955J4888gjh4eGEhYXx6KOP0r9/f+fqrLbMcT5WtoaxRERE6qVFw878+fMBGDt2rMvr77zzDtOmTcNsNrNz507ef/99srKyiImJYdy4cXz66acEBgY673/55Zfx9PTkhhtuoLCwkMsuu4x3330Xs9ncnN1pElp6LiIi0jAtGnYMw6jxuq+vL0uWLKn1c3x8fJg7dy5z585trKa1GtpUUEREpGFaxQRlqV7FBGXN2REREakPhZ1WTkvPRUREGkZhp5VzDGMVl9ooKmn7Z32JiIg0N4WdVi7A4oljb8RcDWWJiIicM4WdVs7Dw0Sgt1ZkiYiI1JfCThsQ7KcVWSIiIvWlsNMGaEWWiIhI/SnstAGOsKNdlEVERM6dwk4b4DwyosDawi0RERFpexR22oCIQAsAJ3OLW7glIiIibY/CThsQGegDQLrCjoiIyDlT2GkDooK8ATiRU9TCLREREWl7FHbaAFV2RERE6k9hpw1oF2iv7CjsiIiInDuFnTYgKshe2TmVV0xpma2FWyMiItK2KOy0AeH+FsweJgwDMvK1/FxERORcKOy0AR4eJiIC7MvP03M0lCUiInIuFHbaCMckZa3IEhEROTcKO22EY/m5JimLiIicG4WdNqKdc/m5KjsiIiLnQmGnjYjU8nMREZF6UdhpIxzLz9M1Z0dEROScKOy0EarsiIiI1I/CThsR6ZigrKXnIiIi50Rhp41wDGOdzCumzGa0cGtERETaDoWdNiLc34LJBGU2g9PaRVlERKTOFHbaCE+zB+H+jnk7mqQsIiJSVwo7bUiU5u2IiIicM4WdNqRiRZYqOyIiInWlsNOGOM7HUmVHRESk7hR22hDH8vMTquyIiIjUmcJOGxIZpMqOiIjIuVLYaUO0i7KIiMi5U9hpQ5xhR+djiYiI1JnCThtSeRdlw9AuyiIiInWhsNOGRATYKzslZQaZBSUt3BoREZG2QWGnDbF4ehDmbwHghIayRERE6kRhp43RJGUREZFzo7DTxjjm7fx4KKOFWyIiItI2KOy0MdcP7QDAv1YfImH/yRZujYiISOunsNPGTB0Yy80jOmIY8OCCrRzLKmzpJomIiLRqCjtt0P9d1Zf+7YPJKijh/g83U1xa1tJNEhERabUUdtogHy8z/7xlCMG+XmxPyeaLbcdbukkiIiKtlsJOGxUX5scdIzsB8MPBUy3cGhERkdZLYacNu7BbOGBfmaUdlUVERKqmsNOGDekYisXTg/TcYg6fym/p5oiIiLRKLRp25syZw/DhwwkMDCQyMpJrrrmGffv2udxjGAazZ88mNjYWX19fxo4dy+7du13uKS4uZubMmURERODv78/UqVNJSUlpzq60CB8vM0M6hgDad0dERKQ6LRp2EhISmD59OuvXr2fZsmWUlpYyYcIE8vMrqhQvvvgiL730EvPmzWPTpk1ER0dz+eWXk5ub67xn1qxZLFq0iAULFrB27Vry8vK46qqrKCtz/1VKI7tGAPDjYYUdERGRqpiMVjTZ4+TJk0RGRpKQkMAll1yCYRjExsYya9YsnnjiCcBexYmKiuKFF17gnnvuITs7m3bt2vHBBx9w4403AnD8+HHi4uL49ttvmThxYq3fm5OTQ3BwMNnZ2QQFBTVpHxvbhsMZ3PjGeiICLGz6/XhMJlNLN0lERKRZ1PXnd6uas5OdnQ1AWFgYAImJiaSlpTFhwgTnPd7e3owZM4Z169YBsHnzZkpKSlzuiY2NJT4+3nnPmYqLi8nJyXF5tFWDOobg7enBqTwrB9LzWro5IiIirU6rCTuGYfDwww8zevRo4uPjAUhLSwMgKirK5d6oqCjntbS0NCwWC6GhodXec6Y5c+YQHBzsfMTFxTV2d5qNt6eZYZ3tfde8HRERkbO1mrAzY8YMduzYwSeffHLWtTOHZgzDqHW4pqZ7nnrqKbKzs52P5OTk+je8FRjZtWIJuoiIiLhqFWFn5syZfPnll6xcuZIOHTo4X4+OjgY4q0KTnp7urPZER0djtVrJzMys9p4zeXt7ExQU5PJoy0aW77ezPjEDm63VTMESERFpFVo07BiGwYwZM1i4cCErVqygS5cuLte7dOlCdHQ0y5Ytc75mtVpJSEhg1KhRAAwdOhQvLy+Xe1JTU9m1a5fzHnc3oEMIfhYzWQUl/JyWW/sbREREziOeLfnl06dP5+OPP+aLL74gMDDQWcEJDg7G19cXk8nErFmzeO655+jRowc9evTgueeew8/Pj5tvvtl575133skjjzxCeHg4YWFhPProo/Tv35/x48e3ZPeajZfZg6GdQllz4BRbkzPpG9u2K1UiIiKNqUXDzvz58wEYO3asy+vvvPMO06ZNA+Dxxx+nsLCQ+++/n8zMTEaMGMHSpUsJDAx03v/yyy/j6enJDTfcQGFhIZdddhnvvvsuZrO5ubrS4npFBbLmwCkOakWWiIiIi1a1z05Lacv77Dgs2JjEkwt3cnGPCD64c0RLN0dERKTJtcl9dqT+ukUGAHBIlR0REREXCjtuons7e9g5nl1EfnFpC7dGRESk9VDYcROh/hbC/S0AHDqp6o6IiIiDwo4bcQxlVTdJ2TAMSstszdkkERGRFqew40a6O+btVFHZMQyDxz/bwYCnl2rFloiInFcUdtyIY95OVWHms80p/HdzCgXWMpbuqfrMMBEREXeksONGulczjJWUUcDsL3c7n285mtWczRIREWlRCjtuxBF2jmYUUFI+N6e0zMZD/9lGvrWM9iG+AGxNykTbK4mIyPmiRXdQlsYVE+yDv8VMvrWMoxn5dI8M5F+rD7P5aCaB3p68f+cFTHplDRn5VpJOF9Ap3B+Aoxn5fLszjaxCKzmFJQT5ePHIhF5YPJWFRUSk7VPYcSMmk4lukQHsSMnmYHoesSG+/CvhEAB/mtqPbu0C6BsbxLbkLLYmZTnDzvSPt7DrWI7LZ/WMCuT6oR3O+g4REZG2Rv90dzOVJyl/vuUYOUWldAr347rB7QEY0jEUgC1Jmc77dh3LwdPDxJ2juzCmZzsAluzWJGYREXEPCjtuxrHXzoH0PN75IRGAX4/qjIeHCYAhnUKAirDzzY5UAEb3iOCPV/Xl8St6AbD6wEkKrWVVfsf/9p7gL1/vITW7sMn6ISIi0ljqFXbee+89vvnmG+fzxx9/nJCQEEaNGsXRo0cbrXFy7hyTlL/flcbhk/kEenvyi2FxzuuOys7e1FwKrKV8veM4AFcNiAWgb0wQHUJ9KSqxkbD/ZJXf8ftFu3hrbSKX/i2B11YepLi06lAkIiLSGtQr7Dz33HP4+tpX9vz444/MmzePF198kYiICB566KFGbaCcG0fYKS61r8a6cXgcAd4VU7Nign2ICvKmzGbw+eYUDqTnYTF7MKFfFGCf93NFv2ig6qGs1OxC0nKKACgsKeOvS/Yx6dU15BSVNGm/RERE6qteYSc5OZnu3bsDsHjxYn7xi1/w29/+ljlz5rBmzZpGbaCcm05hfniZ7UNWHia4Y1Rnl+smk8lZ3Xl5+QEALunZjiAfL+c9E+PtYed/e084l7A7bE/OAqB3dCCv3DiIAG9PDp/MZ9ex7KbojoiISIPVK+wEBASQkZEBwNKlSxk/fjwAPj4+FBZqHkdL8jR70Ll8ldWEvtHEhfmddY8j7JzOtwJw1YCYs65HBFjIKSpl/eEMl2tby8PO4I4hXDO4PX1iAgHIKlBlR0REWqd6hZ3LL7+cu+66i7vuuov9+/dz5ZVXArB79246d+7cmO2TerhqQCxBPp7MuLR7ldcdk5QBvD09GN83yuW62cPE5eWvnTmU5ajsDIqzf0aIn/2k9cwCayO0XEREpPHVK+y89tprjBw5kpMnT/L5558THh4OwObNm7npppsatYFy7h4c34Ptf5pAfPvgKq/3iw12DnWN6xXpMqfHYWL5vJ2lu09gs9l3Wy6zGexMsQ9XDYqzV4dC/ezDX6rsiIhIa1WvTQVDQkKYN2/eWa8//fTTDW6QNA6TyVTtNR8vM0M7hbL+8GmuGRxb5T2jukUQ6O1Jem4xW5IyGdY5jAPpueRby/C3mJ0ToUMdlZ18VXZERKR1qldl5/vvv2ft2rXO56+99hqDBg3i5ptvJjMzs9EaJ03n7zcM4s3bhzkrOGeyeHo4Jyp/uikZqBjC6t8hGHP5vj0Vw1iq7IiISOtUr7Dz2GOPkZNjP15g586dPPLII0yePJnDhw/z8MMPN2oDpWm0D/Hl8r5RNVaAfjXcvj/P1ztSyS0qYVt52BlYPl8HKg9jqbIjIiKtU72GsRITE+nbty8An3/+OVdddRXPPfccW7ZsYfLkyY3aQGk5QzuF0j0ygIPpeXyx7Tjbku3zdQZXCjuaoCwiIq1dvSo7FouFgoICAJYvX86ECRMACAsLc1Z8pO0zmUzO6s77Px5h/4lcoGJyMmiCsoiItH71CjujR4/m4Ycf5s9//jMbN250Lj3fv38/HTropGx3ct2QDljMHuw/kUeZzSAqyJvoYB/ndVV2RESktatX2Jk3bx6enp589tlnzJ8/n/bt7Sdqf/fdd1xxxRWN2kBpWWH+FudRElCxv46Do7KTXVjiXKIuIiLSmtRrzk7Hjh35+uuvz3r95ZdfbnCDpPW56YKOfF1+OvrAM8KOo7JjMyCnqMT5XEREpLWoV9gBKCsrY/HixezduxeTyUSfPn24+uqrMZvNjdk+aQVGdg2na4Q/h0/lc2HXcJdrFk8P/C1m8q1lZBYo7IiISOtTr7Bz8OBBJk+ezLFjx+jVqxeGYbB//37i4uL45ptv6NatW2O3U1qQh4eJ935zAYdP5TvP1aosxM9CvrWQzAIrXfBvgRaKiIhUr15zdh544AG6detGcnIyW7ZsYevWrSQlJdGlSxceeOCBxm6jtAJxYX6M6dmuymuh/uXzdrQiS0REWqF6VXYSEhJYv349YWFhztfCw8N5/vnnueiiixqtcdI2hGpFloiItGL1qux4e3uTm5t71ut5eXlYLJqzcb7RkREiItKa1SvsXHXVVfz2t79lw4YNGIaBYRisX7+ee++9l6lTpzZ2G6WV05ERIiLSmtUr7PzjH/+gW7dujBw5Eh8fH3x8fBg1ahTdu3fnlVdeaeQmSmunjQVFRKQ1q9ecnZCQEL744gsOHjzI3r17MQyDvn370r1798Zun7QBjspOdcNY+cWl5BWXEhXkU+V1ERGRplTnsFPbaearVq1y/vqll16qd4Ok7XFMUK5uGOuOf29k1/FsFk+/iN7RQc3ZNBERkbqHna1bt9bpPpPJVO/GSNsU4qjs5J9d2SkuLWNLUiY2A+Z8+zPv/eaC5m6eiIic5+ocdlauXNmU7ZA2rKbKTvLpAhxHZiXsP8naA6cY3SOiOZsnIiLnuXpNUBapLKSGOTuHT+a7PH/u2706MFRERJqVwo40mGM1VmFJGUUlZS7XEk/Zw87FPSII9PFkT2oOi7Yea/Y2iojI+UthRxosyMcTs4d9rlbWGdWdIxn2sDO4Yyj3j7Wv1vvb0n1nhSIREZGmorAjDWYymQjxdQxluc7bcQxjdY3w59cXdSYiwJvU7CJ+OpLZ7O0UEZHzk8KONIqKeTuuYccxjNU5wh8fLzOju4cDsPHI6eZtoIiInLcUdqRRVKzIqhjGyi8uJT23GIAu4f4ADOtsPzx2U6LCjoiINA+FHWkUVR0Z4ajqhPtbCC6v/FzQxR52tiZnYi21NXMrRUTkfKSwI42i4jDQispO5SEsh+7tAgjx86KoxMau49nN20gRETkvKexIowj1L6/s5FdUdo6Uh50ulcKOh4eJYZ3OHsr6z0/J/HXJzxSXapWWiIg0rhYNO6tXr2bKlCnExsZiMplYvHixy/Vp06ZhMplcHhdeeKHLPcXFxcycOZOIiAj8/f2ZOnUqKSkpzdgLgYoJylmFZ1d2KocdgAu6hAKwqXySclJGAU9+voPXVh7i7vc3U2hV4BERkcbTomEnPz+fgQMHMm/evGrvueKKK0hNTXU+vv32W5frs2bNYtGiRSxYsIC1a9eSl5fHVVddRVmZfmA2p6qOjDhcTdgZ7pikfCQTm83gzTWHnUdKrN5/kjve2UhuUdUnqIuIiJyrOp+N1RQmTZrEpEmTarzH29ub6OjoKq9lZ2fz9ttv88EHHzB+/HgAPvzwQ+Li4li+fDkTJ05s9DZL1UKrODLCsaHgmWEnvn0wvl5msgtLWJ+YwX9+SgbgsYm9eH3VITYmnub2f2/kP/eMxMuskVYREWmYVv+TZNWqVURGRtKzZ0/uvvtu0tPTndc2b95MSUkJEyZMcL4WGxtLfHw869atq/Yzi4uLycnJcXlIw5y5Gisz3+qcrNw53DXseJk9GNwxBIAnPt9BcamNgR2CuX9sNz757YUE+niyNSmL1ftPNl8HRETEbbXqsDNp0iQ++ugjVqxYwd///nc2bdrEpZdeSnGxfe+WtLQ0LBYLoaGhLu+LiooiLS2t2s+dM2cOwcHBzkdcXFyT9uN8cOY+O4nlVZ2YYB98Leaz7ncMZSWfLgTg3jHdMJlMxLcP5pdD7X8eOkNLREQaQ6sOOzfeeCNXXnkl8fHxTJkyhe+++479+/fzzTff1Pg+wzAwmUzVXn/qqafIzs52PpKTkxu76ecd5wTlAis2m0HiyaqHsBwc++2A/SiJCf0qhiqvHdwegGV7Tjjn7hiGwR8W7+SeD37iZPlGhSIiInXRqsPOmWJiYujUqRMHDhwAIDo6GqvVSmam6zlL6enpREVFVfs53t7eBAUFuTykYRxhx2ZAblFplXvsVDYoLsR5eOhvL+nq/DVAfPsgurXzp7jUxpLdJwD4flcaH65PYsnuE1w/f51zWTvYg5BhGE3SLxERafvaVNjJyMggOTmZmJgYAIYOHYqXlxfLli1z3pOamsquXbsYNWpUSzXzvOTtacavfLhqxb4T7Em1z4PqWk3Y8ff25PGJvfjl0A5cO6S9yzWTycQ1g+yvLd56DGupjee//xkAi6cHSacLuH7+Ot75IZGZn2xl8J+XcenfEzieVdhU3RMRkTasRVdj5eXlcfDgQefzxMREtm3bRlhYGGFhYcyePZvrr7+emJgYjhw5wu9+9zsiIiK49tprAQgODubOO+/kkUceITw8nLCwMB599FH69+/vXJ0lzSc6yIfDp/J56NPtztfOnJxc2T1julV77epB7fn7sv38cOgUf1+2j6MZBUQEePP5fSOZ/vEWdh3L4emv9jjvzyoo4ba3N/Dfe0cRVr7B4fbkLPwsZnpEBTZC70REpK0yGS1Y/1+1ahXjxo076/U77riD+fPnc80117B161aysrKIiYlh3Lhx/PnPf3aZUFxUVMRjjz3Gxx9/TGFhIZdddhn//Oc/z2nScU5ODsHBwWRnZ2tIqwHWH87gk41J7EzJ5vCpfAK8PVn12FgiArzr9Xm/mL+On45WDFHOua4/N13QkbziUp74fAfHMgu5uEcEgzuG8PtFu0jNLmJAh2CenNSb11Ye5IeDGQR4e7LuqUsJ8vFqrG6KiEgrUdef3y0adloLhZ3Gl1tUgqeHR5Urserqg/VH+ePiXQD0jArg2wcuxrOafXcOpufyy9d/dNnnx+H1W4dyRXzVezWJiEjbVdef321qzo60HYE+Xg0KOgBX9Y/By2yfuPzU5D7VBh2A7pGBvPvrC/C3mPEwwfVDOnDlAPvcrjUHtF+PiMj5rEXn7IjUJNTfwr9uG0pGnpWxPdvVev/AuBBWPDqWUptB+xBf/rf3BN/sSGX1gZO1bkcgIiLuS2FHWrVLe1e/hUBVooJ8nL++sGs4XmYTyacLOZpRUO0yeBERcW8axhK35e/tydBO9t21NZQlInL+UtgRt3ZxD/vwV8L+Uy3cEhERaSkKO+LWxpTP9fnx0ClKymwt3BoREWkJCjvi1vrGBBHubyHfWsaWo5m1v+EMhmHw+GfbeXX5gSZonYiINAeFHXFrHh4mRveIAGDNgVP8nJbDi9//zH9/qtvhrwfT8/jPTyn8Y8UBymzn/ZZUIiJtklZjidu7uEc7vth2nH+tPsS8lfbjSUwmGN8nitDyoyWqk5JpP2+rzGaQWWCt927QIiLSclTZEbd3cY8IPExQUmbgZTbh4+WBYcCOY9m1vjcls8D565O5xU3ZTBERaSIKO+L2ooJ8+Ndtw/jz1f1Y/9RlTOxnPzpiR3JWre91VHYATuUp7IiItEUaxpLzwuV9KzYnHNghhC+2HWd7Slat76scdlTZERFpm1TZkfPOwLhgALYlZ1PbObjVDWMZhsFfl9R9orOIiLQcVXbkvNMvNhizh4lTecWkZhcRG+Jb7b3HsqoexjqYnsdrKw/h4+XB9UM64OGhc7dERForVXbkvOPjZaZ3dCAA22uYt1NoLeNUntX5vHJlxzG8VVRi43h24VnvFRGR1kNhR85LAzqEALCtfN6OzWbwtyX7+PfaROc9x7IKXN5zslJlp3LASTyV33QNFRGRBlPYkfPSoPJ5O47Kzoqf05m38iDPfL2H7MISAJIzXSs2p3IrqjzHsxR2RETaCoUdOS8NjAsBYNexHMpsBv9cddB5bfdx+/47jqGquDD7nJ7KlZ3UrCLnrw+fVNgREWnNFHbkvNQjMhA/i5m84lI+3pjElqQs57Vdxxxhxz6MNTguFIDT+VbnYaIaxhIRaTsUduS8ZPYwER9rH8p67pu9AAR42xcn7jyWA1RUdvq3t6/eAnvgATheqbKjsCMi0rop7Mh5y7HfTmFJGR4meGpybwB2lk9aPuYcxvIjvPwMrZO5xdhsBmnZFWEnJbOA4tKyZmy5iIicC4UdOW855u0ATBkYy+T4GACOZBSQU1TirOx0CPWlXaD9ANCTucVk5FuxltkwmcDPYsZmQFJGwVmfLyIirYPCjpy3BlUKO/eN7Uaov4X25RsMbj6S6dxEsEOor/O085N5xaSWz9eJDPSmW7sAAA43wlCWzWbw6vIDrDt4qsGfJSIiFbSDspy3OoT68Zdr4rF4etA7Ogiwz885llXI97vSAPs8nmBfL5fKTpCP/T+bmGBfOoX7sfNYdqPM29mQeJqXl++nazt/VjwytsGfJyIidgo7cl679cJOLs/7dwjm+91pLN1jDzsdQn0xmUwuYcfXywxAbIgPXSL8AUhshOXnRzPsn5GUUUCZzXBOihYRkYbRMJZIJf3b2yctZxbYNxbsEGof1mpXPox1qtIwVkywb0XYaYTKjmOOUKnNcNm0UEREGkZhR6QSR9hx6BDqB0BEpcrO8fKVWLEhvnSNaLw5O5UPHU0+rQnPIiKNRWFHpJLKk5Th7MrOybxiZ9UlNtiHzhH2MHQqr5icopIGfbdjE0OAJIUdEZFGo7AjcobK1R1n2Cmv7JzKLXYeFRET4kugjxeR5dcaOm8npdJZXAo7IiKNR2FH5Az9O1QOO/bKjaOyk1NUyonc8mGsYB+ARpm3Yy21cSKnYqPC6sKOYRh8sjGJfWm59f4uEZHzjcKOyBkqV3YcQ1pBvp5YzPb/XAwDvMwm5947XdvZw05D5u2kZRdhMyqeVzdnZ9W+kzy1cCd/WLyz3t8lInK+UdgROcPAuBACvD2JC/MlxM8LwGX5OUB0sA8e5UvD61PZyS8upaxSuknJsocbz/LPPFpN2FmfmAFUHGUhIiK1U9gROUOwrxdLH7qEL6aPxmSq2OsmIsDi/HVMcMUk5i7lK7K2HM3kq+3Ha102/v2uVIb9ZTm/ff8n52uO+TqOXZ2zCkrILjx7wvPmI5kAZORbMQzjrOsiInI2hR2RKsSG+BLmb3F5rXJlxzFfB6BPTCBgXzo+85OtjHp+BY/+d3uVn/vB+qPc99EWCkvKSNh/kqIS+wGijrDTIyrQGarOHMoqLi1jx7Hs8l/byLfq8FERkbpQ2BGpI5ew47I83Y/3f3MB00Z1pn/7YEwm+GxzCtuSs5z3GIbBS8v288fFu3AUZEptBj+XTzQ+VunQ0bgw+6ToM8POrmM5WEttzucZ5Wd31ST5dAGXv5TA+z8eOae+ioi4E4UdkTpyrMgC+7Lzyi7p2Y7ZU/vx1czRXDu4PQBvr010Xl+65wT/+N8BAB68rAdjerYDYGdKFlCxx06HUF86loedM1dkbT562uV5Rr611jb/d3MKB9LzWLT1WK33ioi4K4UdkTqKqGYY60y/uagLAN/uTOV4ViHFpWU89+1eAO65pCsPXd6TgeXL23ek2IelUipVdhxh58xJypuPZro8z8irPeysPXASgNN1CEYiIu5KB4GK1FHlyk7sGZWdyuLbBzOiSxgbEk/z/o9HCfHz4mhGAe0CvXngsh4A9O8QAtjDTmmZjbTyPXY6hPo5w07lYSzDMJxhJ8zfwul8a63DWNmFJc6hNIUdETmfqbIjUkeuE5SrDzsAd462V3c+3nCUeSsOAvD4xF74e9v/fTGgvLJzID2XxFP5lNkMLGYP2gV4VzmMlXS6gFN5VixmD0Z3jwBqH8b68dAp5949uUWllJTZarxfRMRdKeyI1FH78qMjQv28CPKtuSh6WZ8oOoX7kVNUSl5xKQM6BHP9kA7O61FBPkQGemMz7PN5AGJD7Hv3dAy3h51jmYWUlgeUn8qXnMe3D3JWlWobxlp94JTL80xVd0TkPKWwI1JHMcG+zL1pMPNvHeqy/05VzB4mfj2qs/P5/13V17kJoYOjuvPdrlSgIkxFBfpgMXtQajNILT9hfXOSPewM6xxGePmS+Iz86oexDMNg9f6TLq+dLlDYEZHzk+bsiJyDKQNj63zvjcM7svHIaXpHBzGsc9hZ1/u3D2H53nR2HcsBoEOIvaLj4WGiQ5gvh0/mk3y6gLgwP+dmgkM6hlJgLQVqruwczSggJbMQL7OJyEAfjmUVat6OiJy3FHZEmoivxcw/bxla7fUBlQ4chYoT1gE6hvlx+GQ+R08X0K+whP3p9v14hnYKZU+qPRzVNGdnTfkqrGGdwiizGRzLKiQz/+wdmUVEzgcKOyItpP8ZYad9pbDTqXyS8qbE0yzckoJhQOdwP9oFehOeUz6MVcNqLMd8nYt7RrDduSKr9k0IRUTckcKOSAuJCPCmfYgvx7Ice+z4Oa85dlFeWL4ZYIC3J3+a2g+A8PLjJE7nW7HZjLPmApWU2fjxkP3A0Et6tCP5dGH5/arsiMj5SWFHpAX1bx9cKey4DmM5DOgQzNybBtMp3H66uuPMrlKbQU5RCSF+FrILS5i1YCsHT+ZxKtdKYUkZYf4W+sYEEeZvP7k9UxOUReQ81aKrsVavXs2UKVOIjY3FZDKxePFil+uGYTB79mxiY2Px9fVl7Nix7N692+We4uJiZs6cSUREBP7+/kydOpWUlJRm7IVI/TmGsjw9TEQFVezKPLpHBJf1jmTmpd357N5RzqAD4O1pJtDH/u8Ux7yd/+09wcp9J0k+XUhh+eGiN1/QEQ8PE6F+Fpd7RUTONy0advLz8xk4cCDz5s2r8vqLL77ISy+9xLx589i0aRPR0dFcfvnl5ObmOu+ZNWsWixYtYsGCBaxdu5a8vDyuuuoqysp0IrS0foPiQgD7sJW50nCUn8WTt6cN55EJvbB4nv2fqXP5efmKrEMn8wC4ckAMCY+NZdfTE3l0Yi/7veXDXtpnR0TOVy06jDVp0iQmTZpU5TXDMHjllVf4/e9/z3XXXQfAe++9R1RUFB9//DH33HMP2dnZvP3223zwwQeMHz8egA8//JC4uDiWL1/OxIkTm60vIvUxsms4j03s5Qw9dRUe4M2RjALnJOXDJ/MB+9L0ylUgwFnZqcvS89IyG9/tSmNE1zAiA6s//0tEpC1ptZsKJiYmkpaWxoQJE5yveXt7M2bMGNatWwfA5s2bKSkpcbknNjaW+Ph45z1VKS4uJicnx+Uh0hI8PExMH9edi8qPgKirio0FXSs73dr5n3WvY45PXcLO4m3HmfnJVq6e9wOHyz9TRKSta7VhJy0tDYCoqCiX16OiopzX0tLSsFgshIaGVntPVebMmUNwcLDzERcX18itF2lajqGpjDwrZTaDIxn2c7S6tQs4615n2CmwYhhGjZ+7/rB9FVdqdhE3vrGeAydya7xfRKQtaLVhx+HMbfkNw6h1q/7a7nnqqafIzs52PpKTkxulrSLNJdzffihpRn4xxzILsZbasHh6VHkauyPsWEttFFhrnsu2pfxYihA/L07mFnPjG+vZl6bAIyJtW6sNO9HR0QBnVWjS09Od1Z7o6GisViuZmZnV3lMVb29vgoKCXB4ibYmzspNv5dAp+3BTl3B/l0nODr5eZrzLJznXNJSVVWB1zv1ZeN8o4tsHcTrfyqP/3V5rRUhEpDVrtWGnS5cuREdHs2zZMudrVquVhIQERo0aBcDQoUPx8vJyuSc1NZVdu3Y57xFxR2H+FbsoOwJK1yrm64C9Ohpeh3k7W5Oy7J8T4U/XdgG8++sL8LOY2Xksm+V70xux9SIizatFw05eXh7btm1j27ZtgH1S8rZt20hKSsJkMjFr1iyee+45Fi1axK5du5g2bRp+fn7cfPPNAAQHB3PnnXfyyCOP8L///Y+tW7dy66230r9/f+fqLBF3FBFQPoyVZ3VOTq4u7ACEVpq3Ux3HENbgjqHO75hWfnL7S8v2Y7OpuiMibVOLLj3/6aefGDdunPP5ww8/DMAdd9zBu+++y+OPP05hYSH3338/mZmZjBgxgqVLlxIYGOh8z8svv4ynpyc33HADhYWFXHbZZbz77ruYzeZm749Ic6k8jOVYNdU14uzJyQ7OSco1nJTuCDtDOoU4X7v74q68/+NR9qbmsGR3GpP6xzS06edkz/Ec5q44wCMTetI9MrD2N4iIVKFFKztjx47FMIyzHu+++y5gL7/Pnj2b1NRUioqKSEhIID4+3uUzfHx8mDt3LhkZGRQUFPDVV19pdZW4PccE5cwCKwfTy5edR9Yedqo7MqLMZrCtfBhrSMeK1Y2h/hZ+c1FnAF5e3vzVnffWHeG7XWl8tCGpWb9XRNxLq52zIyLVC/Wzn3dlGHCqvFpT4zBWLRsL7j+RS761DH+LmZ5RrhWUO0d3JdDHk/0n8nhy4Q4Wbklhz/GcZpm07BiiO3Iqv8m/S0Tcl8KOSBvkafZwBh6wz68J8vGq9v7aKjuOyckD40LOWtEV7OfFPZd0BeA/P6Xw8H+2M/kfa3hp2f6GdKFOHGHnaPk+QiIi9aGwI9JGhZdPUoaaqzpQefVW1WHHOV+nY2iV16eP684rNw7i9pGdGFh+tMXCLceatLpzOt9KZkEJAMmZBZSW2Zrsu0TEvSnsiLRRjgADVR8TUdW91VV2qpqcXJnJZOKawe155up4Ftx9Id6eHhzLKuTnJtxwsPJxFSVlBseziprsu0TEvSnsiLRREQEVYaemlVhQ85ydypsJDo6rurJTma/FzMU97Gd5/W/viTq391w52uRwJEPzdkSkfhR2RNoox4osqPswVlVhx1HV6Rrh79yPpzaX9bHvUL6sCTcbPHTGQaQKOyJSXwo7Im2U6zBWzZUdx71ZhSWUnbF8fMkue3VmRNfwOn/3Zb0jAdienEV6bsXw0snc4kabx+MIOwHe9u3AjpzSJGURqR+FHZE2yjGM5WU20SH07ANAKwuptFQ9u7DE+XpRSRnf7koFYOrA2Dp/d2SQDwM7BAOwory68/baRIY/u5y/L63bKq0TOUX8+es9JJ+uOsQ4hrEu6WkfMjuqyo6I1JPCjkgb5TgyolO4P57mmv9T9jJ7EORjr5Cczi92vr7y53Ryi0qJCfZhRJewc/r+8eVDWcv3prPhcAbPfbsXgDfWHOZ4VmGt7//3D4m8vTaRhz7ddlY1yFpq42h5CLq0t/17EhV2RKSeFHZE2qjRPSKY2C+KBy7rUaf7HUvVT+dXVHYWbT0GwNWD2uNRxYnpNXHM21l78CQzP9lKmc3AYvbAWmrj1eUHan2/o3Lz09FMlu5xneicdLqAMpuBn8XsDGHJ5a+JiJwrhR2RNirQx4t/3TaszsNPjk0IHZOUswqsrNp3EoBrBtd9CMuhT0wgscE+FJXYSM8tpmdUAP+eNhyA/25OPmuC8ZkqD0u98N3PlFTaR8fx3m7tAmgf4ovF06N8+XntFSMRkTMp7IicJ85ckfXtzjSsZTZ6RwfSOzronD/PZDIxvq+9uuNvMTP/1qGM7hHB+D6R2Ax4qYa5OzabQVL5MJWPlweHT+Xz6aZk53VH1adrO388PEx0DPMDKlZkFZWU8cPBU6r0iEidKOyInCfO3Fhw8Tb7ENY1g9vX+zPvGt2Vsb3a8c9bhzpXhD06sRcmE3yzM5WdKdlVvi89t5iiEhtmDxOPTugFwCvL95NXXAq4VnYAOoeXh53yM7Ke/+5nbnlrA89+s7febReR84fCjsh5IrTSkRHrDp1iY+JpTKZzW4V1po7hfrz76wsY07Od87Xe0UFcXf6Zj3++g0Jr2VnvcwxhdQj15faRnekc7sepPCvzVx0EKnZPduwf1Dnc/r9HMgrILy7lvz/Zq0DvrEtk89HMerdfRM4PCjsi54mw8l2UP9xwlJvf3ADARd0iiA2pedl6fTw1uQ8RARb2pubwh8W7zlpt5Vhp1THMD4unB09N7gPAvxIOs/9ELofKh7EclZ1OEfawczQjn693HCe/PEAZBjz5+Q6KS88OVCIiDgo7IucJx1J1a6kNi6cHNwzrwEs3DGyS74oK8uEfNw3GwwSfb0nh441JLtcdlZ1O5cNTE/pGMb5PFKU2gwc+2Up2YQkmE3QpDzldyis7iafy+WSjvapz75huRARYOJCex2srD3E0I5/XEw7x+GfbOZlbjIiIg2dLN0BEmselvSO5ZlAs3SMDuOmCji6npjeFUd0iePyK3jz/3c88/eUeBrQPoX/5RoRHM+yVHcfwlMlk4pmr+/HjoVPOw0Xbh/ji42UGKkLR4VP5GAZ4epi4c3QX+rcPZvrHW5i74gD/+F/FcvfwAG+euKJ3k/ZPRNoOVXZEzhOh/hZe+dVgZlzao8mDjsM9l3Tl8r5RWMtsvLX2sPN1R9hxrLICiA3x5ZHyycrgegRGbIgvFrMHjtGwy/tG0S7Qm8n9o5nYLwrDALOHyVkJ0jweEalMYUdEmozJZGLaqM4AbEw8jWEYGIbhXELeOcL1ANM7RnVmQHn1p0dkRdgxe5iIC6uYW/SrCzo6P//VXw3mnWnD2fi7y3jz9qEA7EjJctm3p6W8sfoQV7yyWsNqIi1MYUdEmtTgjiF4ephIzS4iJbOQrIIScovsS8wrV3bAHmr+ecsQ7rmkK3dd3NXlmmPIq32ILxd3j3C+7uNlZlzvSMIDvOkaEUCwrxdFJTb2puY0cc9q98nGZH5Oy2XtwZMt3RSR85rCjog0KT+LJ/3a26s1m46cdq7Eig7ycc7JqaxDqB9PTe5DdLCPy+vDOtuPjfj1RZ2rPdrCw8PEkI4hQMsPZZXZDFIy7X1NPq2dn0VaksKOiDS5CzqHArDpSKZzJVbHcL+a3nKWuy7uwjcPjObO0V1qvG9oJ/t3tXTYOZ5VSEmZfZJRdSe7i0jzUNgRkSY3vLwqs+nI6Uorsc4t7HiZPegXG4zJVPOBpUM62sPO1qSsc29oI0qqFHCSMxV2RFqSwo6INDnHENTB9Dy2JtkrLp3C/Wt6S70NjAvBwwTHsgpJzW7c4aNTecUk7D951iaJVXGEOtAwlkhLU9gRkSYX5m+he/nqqtUHTgFnT05uLP7envSJsR9suuVoVqN+9u8W7uSOf2/ks80ptd579HTFqe5pOUWUtoLVYSLnK4UdEWkWjqEsx0nlnZuosgM1z9vZnpzFI//ZzqKtKRSV1P2YCZvN4MdDGQC8ueZwrdWdo6cqKjtlNoPU7KI6f5eINC6FHRFpFhd0CXV5fq4TlM+FY97OlqSzw87/fbmbz7ek8NCn2xn+7HL+uHgX2QUltX7mgfQ8cstPZd9/Io815RUqsAeof/zvgMvePkfPmJSsScoiLUdhR0SahaOyAxDq50Wwr1eTfZejsrP7eLZL9SbxVD7bk7Mwe5joEOpLblEpH6w/yi1vryerwFrjZ249Izi9tTYRgKSMAm59ewMvLdvPNztSATAMg6RKJ7uDJimLtCSFHRFpFh1C/Ygt3zunYxMOYdm/y5d2gd6UlBnsPJbtfH3x1mMAjO4ewerHxvH+by4g3N/CrmM53PLWhhoDj6NKNGVgLCYTrN5/kl3Hsrn/483OTRJ/OnoagIx8K/nWMkwmGNk1HNAkZZGWpLAjIs1meBd7dedcl52fK5PJxNDyoayvth8H7NWWL7bZw861g9vj4WHikp7t+PjuCwn3t7D7eA43v1l94NlSvpT96oGxTOgbBcAtb21g17EcHKvhHROiHSuxooN8nBOzz7Wyk5JZwPrDGef0HhGpmsKOiDSb20d2omOYH9cObt/k33XrhZ0A+GhDEvvSctmeks2RjAJ8vcxcXh5WAHpFB/LJby8kIsDCntQc7nhnE3nlc3McsgtKOJieB8CgjiHOoyyyC+1zfeZc2x+An9NyyC8uJal8JVbHMD/iyledncucna1JmUx+dQ2/emM9Px05XZ/ui0glCjsi0myGdgpj9ePjGNsrssm/a3SPCCb0jaLMZvDM17udQ1gT+kXh7+3pcm/PqEA+vvtCQvy82J6cxW/f/8llrs/WZMfeQH5EBHgzrFMog+JCAJg+rhu/uqAjscE+2AzYkZJdaeNEf+JCy8NOZt2GsTYmnubWtzaQUz409vGGpPr/JogIoLAjIm7sD1f2xeLpwQ8HM5yh4ZpBVVeVekYF8t6vL8DfYmbdoQxmfLzVubrKsRuzY5WXyWTi9VuH8vqtQ3j48l4ADO5UsQIsqTzsdAz3c57WfjK3uNal7usOneKOf28k31pGr6hAAL7ZmeqsIIlI/SjsiIjb6hjux90X28/SspbZCPO3MLpHRLX3D4wL4a07hmPx9GD53hPMW3EQqJic7DhkFCA62Icr4mMwlx9KOri80rM1KdO57LxTuB/Bvl4ElleSUmqYt2MYBk98voPCkjLG9GzHFzMuomdUAMWlNr4sn3ckIvWjsCMibu3+sd2JDrKvArtqQAxe5pr/2hvZLZy//mIAAK+tPMiuY9lsS84CYHDH0GrfN8RZ2clyHnbaKcwfk8lEB+e8neqHsvak5pB8uhAfLw/m3zoEHy8zNw7vCMCnmzSUJdIQCjsi4tb8vT2Ze/NgrhwQw/1ju9fpPVcPas/k/tGU2gzueu8ncotK8fUy0zs6sNr39IsNwmL24HS+lVN59hVdjo0T4+qw187S3ScAuKRHO/ws9krQtYPbYzF7sOtYDrsqLaEXkXOjsCMibm945zBeu3kI0eX7/NTFM1fHE+ZvIS3HfszDgA7BeNZQFfL2NNOvfZDzeUiljRPrsiJr6R572JnQL9r5Wpi/hQn97CvHPt2UXOe2i4grhR0RkSpEBHjzzNX9nM8dw1Q1GVJpmKtTpYNOnZWdaoaxkk8XsDc1Bw8TXNbbdaXajcPjAPh8Swq3vb2B297ewGP/3a7jJ0TOgcKOiEg1rhoQy3VD2mMy4dxIsCaVw07lXaKdlZ1qhrGWlVd1LugSRqi/xeXaRd0i6BzuR4G1jDUHTrHmwCn+uzmFCS+v5s3Vh5v0NPW84lJyirQSTNo+z9pvERE5f/39lwN5emo/An1qP8trSKcQ569dKju1DGMt3ZMGwOV9o8+65uFh4qO7L2TD4QxMJjAMWLApmY2Jp3n22728vTaRiEALvl5m+sYE8eSkPvhazLW2df6qQ5zOL+apSX3wKF9RVllOUQmTXllDUUkZX84cTfsQ31o/U6S1UtgREamByWSqU9ABiAn2JTrIh7ScIpdT3R2HgeYUlZJdWOJyCGpmvpWNifZdkqurHrUP8eW6IR2cz68Z1J7PNqfw7Ld7Scspcs4r2nQkk8SMAt68fSjentUHnlX70nnh+58BuCI+xnlwamVz/3eAY1n2YbdH/7Odj+4aUWUoEmkLNIwlItKI7h3TlYEdgl3m3vhZPIkIsA9PrTt4yuX+//2cjs2APjFBzgpQbTw8TNwwPI41T4xjwW8v5J1fD+eF6/vj62Vm9f6TzFqwrdrhraKSMv7vi93O56v3nzzrnoPpebzzwxEAvMwmfjycwb9/SKxT20RaI4UdEZFGNO2iLnwxYzThAd4ur/eJsa/Uuu+jLdz13k8s2Z3G35bs45Xl+4G6zQk6U5CPFxd2DWdcr0huHN6RN24fisXswXe70nhy4U5sNuOs98xfdYik0wXOw0sTzgg7hmHw56/3UGozuKx3JE9PjQfgxe/3sS8t95zbKNIaKOyIiDSDl28cxM0jOmL2MLF87wnu+WAz81YeJCWzEC+ziSkDYxr8HRf3aMc/bhqM2cPEZ5tTeObrPRhGReBJPJXP/IRDgP0oDYDtKVlk5lec9L7i53QS9p/Ey2ziD1f15aYL4ri0dyTWMhuzPt1GWRUBSqS1U9gREWkGEQHePHdtf5Y+dAlX9o+hS4Q/1wyK5cXrB7D68XF0j6x+w8JzcUV8NC9eb98B+t11R3hpmb1ytD05iwcXbMVaauPiHhH85qLO9I4OxDBgTfnQWkmZjT9/vQeAO0d3pUuEfQfo56/vT7CvF3tTc1i6O+2s7zyWVciCjUnM/GQrTy3cqQ0QpdXRBGURkWbUrV0Ar90ypEm/4/qhHci3lvJ/X+xm7oqDrNp3kp3lAcTPYuaZq+MxmUyM6dmOn9NySdh3kqkDY1m05RhHMgoI97cw49KK3aYjA324fWQn5q44yL9WH+aK+GhMJhPFpWX89v3NZw2FfbIxiRFdwnhsYi+GdQ5zvl5UUsZ1/1xHiJ8XH9w5wnmumEhTa9WVndmzZ2MymVwe0dEVSzMNw2D27NnExsbi6+vL2LFj2b17dw2fKCJyfrh9ZGcem2g/kX3nsWzMHiauG9Kebx64mC4R9j2AxvRsB8DqAyexltqYu/IAAPeO6UaAt+dZn2fx9GBbchabjtgPRp234iAJ+0/iYbIfkvrAZT2YOjAWTw8TGxJPc/u/N5JbaZ+eHw6eYk9qDusOZbCkigqRSFNp9ZWdfv36sXz5cudzs7liOeWLL77ISy+9xLvvvkvPnj35y1/+wuWXX86+ffsIDGyckrCISFs1fVx3An08STyVzx0jO9M5wt/l+tDOofhZzJzMLea5b/eSfLqQiAALt17Y6azPahfozfVDOvDJxiTeWH2IAG9P5q+yz/+Ze9MQrhxQMefoqcm9ufFf60k6XcCqfSeZMjAWsM8Hcnht5UEmlVeIzpW11MY7PyTSOybIGdhEatKqKzsAnp6eREdHOx/t2tn/j20YBq+88gq///3vue6664iPj+e9996joKCAjz/+uIVbLSLSOtw+sjN/mtLvrKAD9vO8RnYNB+zze8Be1aluU8K7L+6CyQTL96Yz/eMtlNoMrugXzeT+rpshxgT7Mrm/Pfw4zvwyDIOVlcLO7uM5rKpi2Xttsgqs3P7vDcz57mdmfLwFa2nT7SAt7qPVh50DBw4QGxtLly5d+NWvfsXhw4cBSExMJC0tjQkTJjjv9fb2ZsyYMaxbt67GzywuLiYnJ8flISJyPhrTq6IyEhHgzS0jzq7qOHRtF+BcIp94Kp9gXy+euaZfldUZxwGmq35Ox1pq4+e0XI5nF+Hj5cFt5ZWj11YcdFktVpujGflcN38d6w/bN2HMLSpl/eGMau//OS2HrAJrtdfl/NGqw86IESN4//33WbJkCW+++SZpaWmMGjWKjIwM0tLs471RUa57U0RFRTmvVWfOnDkEBwc7H3FxcU3WBxGR1qzyMNB9Y6uv6jj89pJuzl//8aq+RAZWfZL8oA4htAv0JrfYHkgcQ1ijukUw49LuWMwe/HQ0kw2Jp0nNLmTlvnSOZuRX+71ZBVaun/8jh0/mExvsw8U9IoCKc8UqO5iex7R3NnLFK2u49p/rKLSW1dgncX+tes7OpEmTnL/u378/I0eOpFu3brz33ntceOGFAGf9i8IwjFrHgJ966ikefvhh5/OcnBwFHhE5L3UK9+fawe05kVPELSM61nr/0E6hPH5FL0pKDa4f0r7a+zw8TIzvE8UnG5NYuieNn1PtGxKO6x1JVJAPvxzWgY82JHHLWxuce/cE+Xiy5KFLiAk++xyuL7cf51ReMZ3C/fjvPSPZdTybNQdOsXzvCZ652l5dstkMXvj+Z95em0hp+Wcmnsrn1f8d4MlJvats54ETuUQF+xBUxyNBpG1q1ZWdM/n7+9O/f38OHDjgXJV1ZhUnPT39rGrPmby9vQkKCnJ5iIicr16+cRAf330hPl61HyAKcP/Y7jw4vket/7B0DGV9uzONLUn2FVyXlh+jce+Ybvh4eVBmMzB7mAj08SSnqJTHP9tR5dDW4q3HALjtwk5EBvkwqlsEfhYzqdlF7Dpmn4rw2ZYU/rX6MKU2g/F9IvnTFPvGiW+uOcye42dPV9h89DQTXlnNjf9ar7k/bq5NhZ3i4mL27t1LTEwMXbp0ITo6mmXLljmvW61WEhISGDVqVAu2UkREAEZ1C8ffYuZ0vhWbAb2jA52np8eF+bFk1iV8NWM0u5+eyKL7L8Lb04M1B07x0YYkl89JyihgS1IWHiaYWr6yy8fLzCU97ENwy/akUVRSxqvL7UvnH53Qk7fuGM6vL+rCpPhoymwGTy3ccdbuzx/8eBTDgL2pOby19rDz9aKSMt5bd4SdKdoc0V206rDz6KOPkpCQQGJiIhs2bOAXv/gFOTk53HHHHZhMJmbNmsVzzz3HokWL2LVrF9OmTcPPz4+bb765pZsuInLe8/Y0M7bSgajjKv0a7ENo/TsE4+NlpntkAI9fYR9qeu7bvS7zd77YZq/qjOoWQWRQxRwhR+Vo6Z4TfLQhiWNZhUQH+XDXxV2d9zw9tR+BPp5sT8l2rjgDyC4o4btdFSMDry4/QFJGAcWlZdz34Wb+9OVufvH6On48VP0E6JaQll3Eo//drnPKzlGrDjspKSncdNNN9OrVi+uuuw6LxcL69evp1Mk+k//xxx9n1qxZ3H///QwbNoxjx46xdOlS7bEjItJKVD7g9NIzws6Zfj2qMyO6hFFgLWPWp9soKinDMAwWl4edawa7zhG6tHckZg8TP6fl8kr5sRgPju/hMhwXGeTjnK/z0tJ9nMgpAuCL7ccoLrXRKyqQi7qHU1xq4/eLdzL9oy2s3GdfEl9cauPO9zax6cjpBv4uNJ6Xl+3ns80pvPD9zy3dlDalVYedBQsWcPz4caxWK8eOHePzzz+nb9++zusmk4nZs2eTmppKUVERCQkJxMfHt2CLRUSksnG9I4kM9KZrO38Gx4XUeK+Hh4m//XIggT6ebE3KYtaCbWxPyebQyXy8PT2Y2M91PmaIn4XhnUMByC0upWuEP78c2uGsz71peEcGxYWQby3juW/3AvDppmQAbhwex1+u6Y+lfAht+d50vD09ePuOYVzcI4ICaxm/fmeTc85RdbILSs5pGX19FJWU8e3OVMC+G7VWmdVdqw47IiLStgX5eLH8kTF8NWM0nubaf+TEhfnxxm3DsJg9+H53Gne99xMA4/tGEVjFiqkJfSs2NHxkQq8qv8PDw8RfronHZIIvth3nrTWH2X08B4vZg2sHt6dLhD8zx9nPArN4evDm7cO4rE8Ub9w2jJFdw8krLuXWtzawal/6WZ8N8J+fkhn6l2Xc+MZ6sgtKXK41ZghavvcEucWlgL3q9OPhU43yuecDhR0REWlSQT5e+HvXfaeTkd3CeeVXgzCZ4FReMQDXDKp6mfuVA2II9vViZNdwJsVHV3kPQHz7YOfS+r98Y6/uTOgXRai/BYB7x3bjL9fE89m9I7mkfO8hX4uZt6dVVHjueu8nFm5Jcfnc73el8uTnOyi1GWxMPM0v/7WO41mFpGQWMGvBVgY+s5Sp835gW3JWnfqeXx5mqrJoi304z9vT/qO78vEbrdmOlCzWlJ+/1lJMRlPX3dqAnJwcgoODyc7O1jJ0EZFW4v0fj/B/X+wmIsDCuicvw+JZ9b/Pi0rKMHuY8KqlcpRVYGXc31aRWV59+fDOEYwu35ywJtZSG49/tp3F244DcM2gWC7vG42X2cSMj7diLbMxuX80m49mciKnmIgACzlFpS4/3E0muHFYHI9N7EV4gPdZ35F8uoAXl+zjq+3HGdk1nD9f04/ukRXzTzPyihnx3P8otRn8fnIfnv12L7HBPvzw5KX1Ol+sOc38ZCtfbT/OjHHdebT8cNrGUtef36rsiIhIq3T7yM58dNcIPr77wmqDDtiXodcWdMA+x8cxWbljmB+juoXXqR0WTw9eumEQd1/cBYDF244z/eMt/PaDzVjLbEyKj2buTUNYeP9FdI8M4FSeFWupjZFdw/nwzhFcP6QDhgELNiUz5q+reG3lQQqtZZSU2diSlMkzX+3hsr8n8NV2e5j68XAGk15dw4vf/+ycl/P1jlRKbQb92wdz28hO+Hh5cDy7iH0nKlZl2Wx1r12k5xQxb8UB0ssnbDcVa6nNOfx35mq85qTKDqrsiIicLwzD4PtdafSMDqRbu4Bzfv/GxNMs3Z3Gyn3pHDqZzyU92/Hm7UPx9rSvAMsqsPLvtYkM7hjK2F7tnFWXn46cZvZXu50bIIb7WygqKSO/0iTji3tEcOfoLny4/ijL99oDQvsQX2ZP7ce8lQfZnpzFH6/qy52ju/Cbdzex4ud0HpvYi+njuvPJxiSe/+5nZl7a3WXpfXV+/c5GVu47SdcIfz69ZyTtAs+uNjWGHw6e4pa3NhARYGHD78Zj9mjcKlRdf34r7KCwIyIi5+5UXjFhfhY86vgD3GYz+GrHcf66ZB8pmYUAhPh5cWGXcG4a0dHlnLJle04w+8vdHMsqdL5m9jCx/qnLaBfozYfrj/KHxbsY2imU+8d24+73f8JR2Hn91iFcER9TbTs2HTnNL1//0fm8V1QgC357oXP+UlW+35XKS8v2c9fortwwvO7HK83+cjfvrjvCDcM68OIvBtb5fXWlsHMOFHZERKS5FJeW8eOhDKKCfOgVFVhtWCqwljJvxUHeXHOYkjKDsb3a8e6vLwDgWFYhFz2/Ag+TfRivwFpGXJgvyacL8fUy8997RxLfPviszzQMgxv/tZ6NR04zvk8UO1KySM8tpn/7YD66e0SVZ4TlF5dyyYsryci3nyD/y6EdeObqePKKS/l+dxqJJ/N5cHwPgn29zvqu0S+s5FhWIW/ePozL+9Z8lFN91PXnd6s+CFRERMTdeHuaGdur9vkrfhZPHr+iN9cNac+X245z4wUVB7W2D/Gld3QgP6flUmAt4+IeEbx5+zB++8FmVu8/yd3v/8QX0y9y2XEaIGH/STYeOY3F04M/X9OPvKJSbnxjPTuPZXPLmxt47zcXEHZGhefddUfIyLcS4udFTmEJ/92cwqr9J8nIK3ZWk/y9zTwywXXy8c9puRzLKsTb04PR3WufCN6UNEFZRESkFeseGcjDE3o5zxVzcOxO3SsqkNduGYKPl5l5Nw+mWzt/UrOLeObrPS7322wGf12yD4A7RnYiJtiXHlGBfHCnPeDsPJbNjf/60WXScnZBCa8nHALsR298eNcIIgIsnMy1Bx1Hm1aV7zpd2fI9JwD7XCRfS90OmW0qCjsiIiJt0L1ju/Hctf35uNLwU5CPF3NvGoLJZF/BVfm09692HGf38RwCvD25b2x35+v9YoP5zz0XEh3kw4H0PH7x+o/8VH5ExhtrDpFbVEqvqECmDIhlVLcIvnvwEl78xQDWPD6ORdPtB2/vPJbt3BPJYdlee9gZ36fxh6/OlcKOiIhIG+Rn8eTmER3P2renb2wQVw2wnw7/0jJ7JSctu4g/fbkbgLsv7nrWUFX3yED+e+9IOob5kXS6gF+8/iO/mL+Of689AsAjE3o65xa1C/TmhmFxxIX5ERnoQ79Y+1yZ1fsrqjtp2UXsSMnGZILLFHZERESksc0a3wMPEyzfm87mo5k8/J9tZBWUEN8+iPvGdqvyPXFhfnx230h+NTwOi9mDn45mUlhSxsC4kBonF4/tZV9FVnko638/26s6g+JCmmxZ+7lQ2BEREXEz3doFcP0Q+6Gov3l3E+sOZeDrZebVXw2ucYPGyEAfnr9+AGueGMc9l3RlUFwIz14TX+MuzWN62idbrzlwkjKbgc1msGCj/aDV1jCEBVqNJSIi4pYeuKwHi7cdI7vQfjzG7Kl967yRYlSQD09N7lOne4d0DCHQx5PMghJ2pGRxNKOAnceyCfD25MZz2JOnKamyIyIi4obiwvy4ZUQnACb3j+aGYU0TPDzNFUvLl+w+4Vzxdd/YbkRUcQ5YS1BlR0RExE39bnIfLu4RwegeEU16YOjYXu34blcab645TJnNICbYh99c1KXJvu9cKeyIiIi4KYunR7Oshrqk/KiLsvJdBh+d0KvF99apTMNYIiIi0iAxwfYdnQH6xgRx7eD2LdwiVwo7IiIi0mB3X9yVzuF+PHttfJ0PR20uGsYSERGRBrt+aAeuH9qhpZtRJVV2RERExK0p7IiIiIhbU9gRERERt6awIyIiIm5NYUdERETcmsKOiIiIuDWFHREREXFrCjsiIiLi1hR2RERExK0p7IiIiIhbU9gRERERt6awIyIiIm5NYUdERETcmsKOiIiIuDXPlm5Aa2AYBgA5OTkt3BIRERGpK8fPbcfP8eoo7AC5ubkAxMXFtXBLRERE5Fzl5uYSHBxc7XWTUVscOg/YbDaOHz9OYGAgJpOppZvTqHJycoiLiyM5OZmgoKCWbk6TU3/d1/nUV1B/3d351N+m7KthGOTm5hIbG4uHR/Uzc1TZATw8POjQoUNLN6NJBQUFuf1/UJWpv+7rfOorqL/u7nzqb1P1taaKjoMmKIuIiIhbU9gRERERt6aw4+a8vb3505/+hLe3d0s3pVmov+7rfOorqL/u7nzqb2voqyYoi4iIiFtTZUdERETcmsKOiIiIuDWFHREREXFrCjsiIiLi1hR23MCcOXMYPnw4gYGBREZGcs0117Bv3z6XewzDYPbs2cTGxuLr68vYsWPZvXt3C7W4cc2ZMweTycSsWbOcr7lbf48dO8att95KeHg4fn5+DBo0iM2bNzuvu1N/S0tL+cMf/kCXLl3w9fWla9euPPPMM9hsNuc9bbW/q1evZsqUKcTGxmIymVi8eLHL9br0q7i4mJkzZxIREYG/vz9Tp04lJSWlGXtRdzX1t6SkhCeeeIL+/fvj7+9PbGwst99+O8ePH3f5DHfp75nuueceTCYTr7zyisvr7tbfvXv3MnXqVIKDgwkMDOTCCy8kKSnJeb25+quw4wYSEhKYPn0669evZ9myZZSWljJhwgTy8/Od97z44ou89NJLzJs3j02bNhEdHc3ll1/uPBesrdq0aRNvvPEGAwYMcHndnfqbmZnJRRddhJeXF9999x179uzh73//OyEhIc573Km/L7zwAq+//jrz5s1j7969vPjii/z1r39l7ty5znvaan/z8/MZOHAg8+bNq/J6Xfo1a9YsFi1axIIFC1i7di15eXlcddVVlJWVNVc36qym/hYUFLBlyxb++Mc/smXLFhYuXMj+/fuZOnWqy33u0t/KFi9ezIYNG4iNjT3rmjv199ChQ4wePZrevXuzatUqtm/fzh//+Ed8fHyc9zRbfw1xO+np6QZgJCQkGIZhGDabzYiOjjaef/555z1FRUVGcHCw8frrr7dUMxssNzfX6NGjh7Fs2TJjzJgxxoMPPmgYhvv194knnjBGjx5d7XV36++VV15p/OY3v3F57brrrjNuvfVWwzDcp7+AsWjRIufzuvQrKyvL8PLyMhYsWOC859ixY4aHh4fx/fffN1vb6+PM/lZl48aNBmAcPXrUMAz37G9KSorRvn17Y9euXUanTp2Ml19+2XnN3fp74403Ov+7rUpz9leVHTeUnZ0NQFhYGACJiYmkpaUxYcIE5z3e3t6MGTOGdevWtUgbG8P06dO58sorGT9+vMvr7tbfL7/8kmHDhvHLX/6SyMhIBg8ezJtvvum87m79HT16NP/73//Yv38/ANu3b2ft2rVMnjwZcL/+OtSlX5s3b6akpMTlntjYWOLj49t03x2ys7MxmUzOqqW79ddms3Hbbbfx2GOP0a9fv7Ouu1N/bTYb33zzDT179mTixIlERkYyYsQIl6Gu5uyvwo6bMQyDhx9+mNGjRxMfHw9AWloaAFFRUS73RkVFOa+1NQsWLGDLli3MmTPnrGvu1t/Dhw8zf/58evTowZIlS7j33nt54IEHeP/99wH36+8TTzzBTTfdRO/evfHy8mLw4MHMmjWLm266CXC//jrUpV9paWlYLBZCQ0OrvaetKioq4sknn+Tmm292Hhbpbv194YUX8PT05IEHHqjyujv1Nz09nby8PJ5//nmuuOIKli5dyrXXXst1111HQkIC0Lz91annbmbGjBns2LGDtWvXnnXNZDK5PDcM46zX2oLk5GQefPBBli5d6jL2eyZ36a/NZmPYsGE899xzAAwePJjdu3czf/58br/9dud97tLfTz/9lA8//JCPP/6Yfv36sW3bNmbNmkVsbCx33HGH8z536e+Z6tOvtt73kpISfvWrX2Gz2fjnP/9Z6/1tsb+bN2/m1VdfZcuWLefc9rbYX8eCgquvvpqHHnoIgEGDBrFu3Tpef/11xowZU+17m6K/quy4kZkzZ/Lll1+ycuVKOnTo4Hw9Ojoa4KyknJ6efta/ItuCzZs3k56eztChQ/H09MTT05OEhAT+8Y9/4Onp6eyTu/Q3JiaGvn37urzWp08f54oGd/vzfeyxx3jyySf51a9+Rf/+/bntttt46KGHnFU8d+uvQ136FR0djdVqJTMzs9p72pqSkhJuuOEGEhMTWbZsmbOqA+7V3zVr1pCenk7Hjh2df28dPXqURx55hM6dOwPu1d+IiAg8PT1r/burufqrsOMGDMNgxowZLFy4kBUrVtClSxeX6126dCE6Opply5Y5X7NarSQkJDBq1Kjmbm6DXXbZZezcuZNt27Y5H8OGDeOWW25h27ZtdO3a1a36e9FFF521lcD+/fvp1KkT4H5/vgUFBXh4uP7VZDabnf9SdLf+OtSlX0OHDsXLy8vlntTUVHbt2tUm++4IOgcOHGD58uWEh4e7XHen/t52223s2LHD5e+t2NhYHnvsMZYsWQK4V38tFgvDhw+v8e+uZu1vo053lhZx3333GcHBwcaqVauM1NRU56OgoMB5z/PPP28EBwcbCxcuNHbu3GncdNNNRkxMjJGTk9OCLW88lVdjGYZ79Xfjxo2Gp6en8eyzzxoHDhwwPvroI8PPz8/48MMPnfe4U3/vuOMOo3379sbXX39tJCYmGgsXLjQiIiKMxx9/3HlPW+1vbm6usXXrVmPr1q0GYLz00kvG1q1bnauP6tKve++91+jQoYOxfPlyY8uWLcall15qDBw40CgtLW2pblWrpv6WlJQYU6dONTp06GBs27bN5e+u4uJi52e4S3+rcuZqLMNwr/4uXLjQ8PLyMt544w3jwIEDxty5cw2z2WysWbPG+RnN1V+FHTcAVPl45513nPfYbDbjT3/6kxEdHW14e3sbl1xyibFz586Wa3QjOzPsuFt/v/rqKyM+Pt7w9vY2evfubbzxxhsu192pvzk5OcaDDz5odOzY0fDx8TG6du1q/P73v3f5AdhW+7ty5coq/1u94447DMOoW78KCwuNGTNmGGFhYYavr69x1VVXGUlJSS3Qm9rV1N/ExMRq/+5auXKl8zPcpb9VqSrsuFt/3377baN79+6Gj4+PMXDgQGPx4sUun9Fc/TUZhmE0bq1IREREpPXQnB0RERFxawo7IiIi4tYUdkRERMStKeyIiIiIW1PYEREREbemsCMiIiJuTWFHRERE3JrCjoiIiLg1hR0RaRGdO3fmlVdeqfP9q1atwmQykZWV1WRtEhH3pB2URaROxo4dy6BBg84poNTk5MmT+Pv74+fnV6f7rVYrp0+fJioqCpPJ1ChtOFerVq1i3LhxZGZmEhIS0iJtEJFz59nSDRAR92EYBmVlZXh61v5XS7t27c7psy0WC9HR0fVtmoicxzSMJSK1mjZtGgkJCbz66quYTCZMJhNHjhxxDi0tWbKEYcOG4e3tzZo1azh06BBXX301UVFRBAQEMHz4cJYvX+7ymWcOY5lMJt566y2uvfZa/Pz86NGjB19++aXz+pnDWO+++y4hISEsWbKEPn36EBAQwBVXXEFqaqrzPaWlpTzwwAOEhIQQHh7OE088wR133ME111xTbV+PHj3KlClTCA0Nxd/fn379+vHtt99y5MgRxo0bB0BoaCgmk4lp06YB9pD34osv0rVrV3x9fRk4cCCfffbZWW3/5ptvGDhwID4+PowYMYKdO3fW+r0i0nAKOyJSq1dffZWRI0dy9913k5qaSmpqKnFxcc7rjz/+OHPmzGHv3r0MGDCAvLw8Jk+ezPLly9m6dSsTJ05kypQpJCUl1fg9Tz/9NDfccAM7duxg8uTJ3HLLLZw+fbra+wsKCvjb3/7GBx98wOrVq0lKSuLRRx91Xn/hhRf46KOPeOedd/jhhx/Iyclh8eLFNbZh+vTpFBcXs3r1anbu3MkLL7xAQEAAcXFxfP755wDs27eP1NRUXn31VQD+8Ic/8M477zB//nx2797NQw89xK233kpCQoLLZz/22GP87W9/Y9OmTURGRjJ16lRKSkpq/F4RaQSNfo66iLilMWPGGA8++KDLaytXrjQAY/HixbW+v2/fvsbcuXOdzzt16mS8/PLLzueA8Yc//MH5PC8vzzCZTMZ3333n8l2ZmZmGYRjGO++8YwDGwYMHne957bXXjKioKOfzqKgo469//avzeWlpqdGxY0fj6quvrrad/fv3N2bPnl3ltTPb4Ginj4+PsW7dOpd777zzTuOmm25yed+CBQuc1zMyMgxfX1/j008/rfV7RaRhNGdHRBps2LBhLs/z8/N5+umn+frrrzl+/DilpaUUFhbWWtkZMGCA89f+/v4EBgaSnp5e7f1+fn5069bN+TwmJsZ5f3Z2NidOnOCCCy5wXjebzQwdOhSbzVbtZz7wwAPcd999LF26lPHjx3P99de7tOtMe/bsoaioiMsvv9zldavVyuDBg11eGzlypPPXYWFh9OrVi71799bre0Wk7jSMJSIN5u/v7/L8scce4/PPP+fZZ59lzZo1bNu2jf79+2O1Wmv8HC8vL5fnJpOpxmBS1f3GGQtMz1y5deb1M911110cPnyY2267jZ07dzJs2DDmzp1b7f2O9n3zzTds27bN+dizZ4/LvJ3qONp3rt8rInWnsCMidWKxWCgrK6vTvWvWrGHatGlce+219O/fn+joaI4cOdK0DTxDcHAwUVFRbNy40flaWVkZW7durfW9cXFx3HvvvSxcuJBHHnmEN998E7D/Hjg+x6Fv3754e3uTlJRE9+7dXR6V5zUBrF+/3vnrzMxM9u/fT+/evWv9XhFpGA1jiUiddO7cmQ0bNnDkyBECAgIICwur9t7u3buzcOFCpkyZgslk4o9//GONFZqmMnPmTObMmUP37t3p3bs3c+fOJTMzs8Z9embNmsWkSZPo2bMnmZmZrFixgj59+gDQqVMnTCYTX3/9NZMnT8bX15fAwEAeffRRHnroIWw2G6NHjyYnJ4d169YREBDAHXfc4fzsZ555hvDwcKKiovj9739PRESEc2VYTd8rIg2jyo6I1Mmjjz6K2Wymb9++tGvXrsb5Ny+//DKhoaGMGjWKKVOmMHHiRIYMGdKMrbV74oknuOmmm7j99tsZOXIkAQEBTJw4ER8fn2rfU1ZWxvTp0+nTpw9XXHEFvXr14p///CcA7du35+mnn+bJJ58kKiqKGTNmAPDnP/+Z//u//2POnDn06dOHiRMn8tVXX9GlSxeXz37++ed58MEHGTp0KKmpqXz55Zcu1aLqvldEGkY7KIvIecNms9GnTx9uuOEG/vznPzfb92rnZZGWpWEsEXFbR48eZenSpYwZM4bi4mLmzZtHYmIiN998c0s3TUSakYaxRMRteXh48O677zJ8+HAuuugidu7cyfLlyzUXRuQ8o2EsERERcWuq7IiIiIhbU9gRERERt6awIyIiIm5NYUdERETcmsKOiIiIuDWFHREREXFrCjsiIiLi1hR2RERExK39P1LrHQe9ZTWUAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 准备优化器和模型\n",
    "vocab_size = len(dataset.token2id)\n",
    "hidden_size = 32\n",
    "n_layers = 1\n",
    "dropout = 0\n",
    "n_tags = len(dataset.label2id)\n",
    "batch_size = 128\n",
    "epochs = 20\n",
    "learning_rate = 1e-2\n",
    "\n",
    "lstm_crf = LSTM_CRF(vocab_size, hidden_size, n_layers, dropout, n_tags)\n",
    "train(lstm_crf, batch_size, epochs, learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "8a9138a5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precision = 0.5846536191363778, recall = 0.4487362899380067, f1 = 0.5077566437339808\n",
      "precision = 0.5070189925681255, recall = 0.3759951010410288, f1 = 0.4317862165963432\n"
     ]
    }
   ],
   "source": [
    "evaluate(train_X, train_Y, lstm_crf, batch_size)\n",
    "evaluate(test_X, test_Y, lstm_crf, batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6454195f",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
